--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:14:14.305826Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-12b34-9c57c45ff-5kq92 namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: d7ee9fe4-47e4-40a4-a49e-6a2a6d949994 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-12b34-9c57c45ff-5kq92 to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:14:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:14:14Z" name: ensemble-graph-12b34-9c57c45ff-5kq92.18a6ef843e6609d4 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 882dc0a0-6e01-4c60-a87c-38f7371ec100 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:14Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-12b34-9c57c45ff-5kq92 namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: d7ee9fe4-47e4-40a4-a49e-6a2a6d949994 kind: Event lastTimestamp: "2026-04-16T20:14:14Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:14:14Z" name: ensemble-graph-12b34-9c57c45ff-5kq92.18a6ef845732421f namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 41545880-f8d3-4d86-93fe-d61977936184 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-12b34} kind: Pod name: ensemble-graph-12b34-9c57c45ff-5kq92 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: d7ee9fe4-47e4-40a4-a49e-6a2a6d949994 kind: Event lastTimestamp: "2026-04-16T20:14:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:14Z" name: ensemble-graph-12b34-9c57c45ff-5kq92.18a6ef84586ae53b namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: d7b2ca43-7dd4-47ce-b27c-56f000b967a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-12b34} kind: Pod name: ensemble-graph-12b34-9c57c45ff-5kq92 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: d7ee9fe4-47e4-40a4-a49e-6a2a6d949994 kind: Event lastTimestamp: "2026-04-16T20:14:14Z" message: 'Created container: ensemble-graph-12b34' metadata: creationTimestamp: "2026-04-16T20:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:14Z" name: ensemble-graph-12b34-9c57c45ff-5kq92.18a6ef845b6fddab namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: 93b74316-867f-479e-a930-727f57a98a44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-12b34} kind: Pod name: ensemble-graph-12b34-9c57c45ff-5kq92 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: d7ee9fe4-47e4-40a4-a49e-6a2a6d949994 kind: Event lastTimestamp: "2026-04-16T20:14:14Z" message: Started container ensemble-graph-12b34 metadata: creationTimestamp: "2026-04-16T20:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:14Z" name: ensemble-graph-12b34-9c57c45ff-5kq92.18a6ef845bdc74be namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: 8d47c6ec-1979-4307-af29-16953cad8732 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-12b34} kind: Pod name: ensemble-graph-12b34-9c57c45ff-5kq92 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: d7ee9fe4-47e4-40a4-a49e-6a2a6d949994 kind: Event lastTimestamp: "2026-04-16T20:22:28Z" message: Stopping container ensemble-graph-12b34 metadata: creationTimestamp: "2026-04-16T20:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:28Z" name: ensemble-graph-12b34-9c57c45ff-5kq92.18a6eff769bfdd2c namespace: kserve-ci-e2e-test resourceVersion: "23648" uid: e3e2ea24-18f8-4cff-a55a-05af4270d009 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:22:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-12b34} kind: Pod name: ensemble-graph-12b34-9c57c45ff-5kq92 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: d7ee9fe4-47e4-40a4-a49e-6a2a6d949994 kind: Event lastTimestamp: "2026-04-16T20:22:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:56Z" name: ensemble-graph-12b34-9c57c45ff-5kq92.18a6eff80e6c957b namespace: kserve-ci-e2e-test resourceVersion: "23966" uid: 02ed1bfa-32dc-4a51-bf67-a9f8c78e71ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-12b34-9c57c45ff namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 528da354-fe17-4b97-a226-ef7cf703d6b0 kind: Event lastTimestamp: "2026-04-16T20:14:14Z" message: 'Created pod: ensemble-graph-12b34-9c57c45ff-5kq92' metadata: creationTimestamp: "2026-04-16T20:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:14Z" name: ensemble-graph-12b34-9c57c45ff.18a6ef843df34471 namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 9abcfe4e-d42e-4e32-b927-49857ba92cb2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-12b34 namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: d7b57619-e585-4571-8c45-5c6e96996761 kind: Event lastTimestamp: "2026-04-16T20:14:14Z" message: Scaled up replica set ensemble-graph-12b34-9c57c45ff from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:14Z" name: ensemble-graph-12b34.18a6ef843d2564a1 namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 85545dc0-e556-4449-9b16-cc5b06c81578 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-12b34 namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 2a167b8d-65bf-407f-9433-020feff034e5 kind: Event lastTimestamp: "2026-04-16T20:14:14Z" message: InferenceGraph [ensemble-graph-12b34] is Ready metadata: creationTimestamp: "2026-04-16T20:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:14Z" name: ensemble-graph-12b34.18a6ef8442e06376 namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 4856e542-80ab-45be-86bd-67d28d45d83c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:14:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-12b34 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: cf17a647-26d5-4038-9513-03e90060d4fe kind: Event lastTimestamp: "2026-04-16T20:14:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:44Z" name: ensemble-graph-12b34.18a6ef87bb710aba namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 71d38d34-973d-4c6a-98e2-a268cbb50cfc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:14:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-12b34 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: cf17a647-26d5-4038-9513-03e90060d4fe kind: Event lastTimestamp: "2026-04-16T20:14:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:44Z" name: ensemble-graph-12b34.18a6ef87bbd5d4b8 namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: c706cd83-c2bd-467e-abee-3d341b724015 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:12:54.000306Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-de58a-7d7f775b78-25zfk namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: f46f208b-213f-4c52-b9d4-b1c0bc3ae4cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-de58a-7d7f775b78-25zfk to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:12:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:12:54Z" name: ensemble-graph-de58a-7d7f775b78-25zfk.18a6ef718bd10d98 namespace: kserve-ci-e2e-test resourceVersion: "19104" uid: e0a881ed-d662-4a44-ac39-db3d617688f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:54Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-de58a-7d7f775b78-25zfk namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: f46f208b-213f-4c52-b9d4-b1c0bc3ae4cf kind: Event lastTimestamp: "2026-04-16T20:12:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-de58a-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:12:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:54Z" name: ensemble-graph-de58a-7d7f775b78-25zfk.18a6ef719a1560e7 namespace: kserve-ci-e2e-test resourceVersion: "19115" uid: 140a5a15-d424-4e34-91a1-08d132dff061 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:55Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-de58a-7d7f775b78-25zfk namespace: kserve-ci-e2e-test resourceVersion: "19108" uid: f46f208b-213f-4c52-b9d4-b1c0bc3ae4cf kind: Event lastTimestamp: "2026-04-16T20:12:55Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:12:55Z" name: ensemble-graph-de58a-7d7f775b78-25zfk.18a6ef71c88473ee namespace: kserve-ci-e2e-test resourceVersion: "19125" uid: da90dbfd-f3ee-4acc-8ffe-0f5262e289a3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de58a} kind: Pod name: ensemble-graph-de58a-7d7f775b78-25zfk namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: f46f208b-213f-4c52-b9d4-b1c0bc3ae4cf kind: Event lastTimestamp: "2026-04-16T20:12:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:55Z" name: ensemble-graph-de58a-7d7f775b78-25zfk.18a6ef71c9f14477 namespace: kserve-ci-e2e-test resourceVersion: "19128" uid: c0e5eb95-3bc3-4057-87bb-7ac054b6991c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de58a} kind: Pod name: ensemble-graph-de58a-7d7f775b78-25zfk namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: f46f208b-213f-4c52-b9d4-b1c0bc3ae4cf kind: Event lastTimestamp: "2026-04-16T20:12:55Z" message: 'Created container: ensemble-graph-de58a' metadata: creationTimestamp: "2026-04-16T20:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:55Z" name: ensemble-graph-de58a-7d7f775b78-25zfk.18a6ef71cd03adfc namespace: kserve-ci-e2e-test resourceVersion: "19129" uid: efbdfd0c-5a8e-4c48-8903-ebe975800b7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de58a} kind: Pod name: ensemble-graph-de58a-7d7f775b78-25zfk namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: f46f208b-213f-4c52-b9d4-b1c0bc3ae4cf kind: Event lastTimestamp: "2026-04-16T20:12:55Z" message: Started container ensemble-graph-de58a metadata: creationTimestamp: "2026-04-16T20:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:55Z" name: ensemble-graph-de58a-7d7f775b78-25zfk.18a6ef71cd6eadd7 namespace: kserve-ci-e2e-test resourceVersion: "19130" uid: 4e53d7b6-2d8f-442d-a3d6-088f59983c81 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de58a} kind: Pod name: ensemble-graph-de58a-7d7f775b78-25zfk namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: f46f208b-213f-4c52-b9d4-b1c0bc3ae4cf kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Stopping container ensemble-graph-de58a metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:04Z" name: ensemble-graph-de58a-7d7f775b78-25zfk.18a6ef73e552a051 namespace: kserve-ci-e2e-test resourceVersion: "19220" uid: 8afa2aa7-7d1b-4eaa-9f6f-dc4b7bd8185f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de58a} kind: Pod name: ensemble-graph-de58a-7d7f775b78-25zfk namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: f46f208b-213f-4c52-b9d4-b1c0bc3ae4cf kind: Event lastTimestamp: "2026-04-16T20:13:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:31Z" name: ensemble-graph-de58a-7d7f775b78-25zfk.18a6ef74732f89ca namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: d2248230-cfb6-4a2e-a6f5-60634a771e4f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-de58a-7d7f775b78 namespace: kserve-ci-e2e-test resourceVersion: "19096" uid: b8075ba0-9200-43b7-b534-26ef72a6ca50 kind: Event lastTimestamp: "2026-04-16T20:12:53Z" message: 'Created pod: ensemble-graph-de58a-7d7f775b78-25zfk' metadata: creationTimestamp: "2026-04-16T20:12:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:53Z" name: ensemble-graph-de58a-7d7f775b78.18a6ef718b609552 namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: f9575188-88f4-45dd-a080-2ec9523529e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-de58a namespace: kserve-ci-e2e-test resourceVersion: "19094" uid: 31ef16d4-cd29-43a6-9ef9-8b4dd7ca1e35 kind: Event lastTimestamp: "2026-04-16T20:12:53Z" message: Scaled up replica set ensemble-graph-de58a-7d7f775b78 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:12:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:53Z" name: ensemble-graph-de58a.18a6ef718a4ef0a4 namespace: kserve-ci-e2e-test resourceVersion: "19097" uid: 5aa95683-3119-4851-927d-219bd6e72f17 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-de58a namespace: kserve-ci-e2e-test resourceVersion: "19110" uid: 30414964-3508-430e-97ed-42b666463d72 kind: Event lastTimestamp: "2026-04-16T20:12:54Z" message: InferenceGraph [ensemble-graph-de58a] is Ready metadata: creationTimestamp: "2026-04-16T20:12:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:12:54Z" name: ensemble-graph-de58a.18a6ef718f293254 namespace: kserve-ci-e2e-test resourceVersion: "19112" uid: e0c7bee2-55dd-407a-a435-b50b11c00df3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:13:04.279973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn namespace: kserve-ci-e2e-test resourceVersion: "19288" uid: b181d931-60a8-48dc-abce-f5911d25b5b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn.18a6ef73f0884ad5 namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: a40728bc-94ff-4e5e-8f5c-e211d1103bab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn namespace: kserve-ci-e2e-test resourceVersion: "19298" uid: b181d931-60a8-48dc-abce-f5911d25b5b9 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn.18a6ef73fb4d40de namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: 39fb3ba8-09d0-4b07-bb7e-44a721489d63 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn namespace: kserve-ci-e2e-test resourceVersion: "19291" uid: b181d931-60a8-48dc-abce-f5911d25b5b9 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn.18a6ef73fcb062cd namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: c6babc64-a08a-45ae-943f-1163f563ad00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn namespace: kserve-ci-e2e-test resourceVersion: "19291" uid: b181d931-60a8-48dc-abce-f5911d25b5b9 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn.18a6ef7400fbc6ca namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 11974ff8-2adb-4867-b4c3-01aea89cfde4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn namespace: kserve-ci-e2e-test resourceVersion: "19291" uid: b181d931-60a8-48dc-abce-f5911d25b5b9 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn.18a6ef74016e33f9 namespace: kserve-ci-e2e-test resourceVersion: "19315" uid: fa1976e0-3e4e-4344-8b80-c32df260a50d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn namespace: kserve-ci-e2e-test resourceVersion: "19291" uid: b181d931-60a8-48dc-abce-f5911d25b5b9 kind: Event lastTimestamp: "2026-04-16T20:13:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:45Z" name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn.18a6ef74138b0fec namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 9f000597-bcc6-45d3-a2a8-94a8954b6dad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn namespace: kserve-ci-e2e-test resourceVersion: "19291" uid: b181d931-60a8-48dc-abce-f5911d25b5b9 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn.18a6eff775bdbdad namespace: kserve-ci-e2e-test resourceVersion: "23724" uid: 046b9a2f-9bd8-4ee5-b2c2-9261267a7785 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-12b34-predictor-558d6f8bf8 namespace: kserve-ci-e2e-test resourceVersion: "19281" uid: 53cc9560-1b54-4f28-9d8a-eb13b8a59170 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'Created pod: error-404-isvc-12b34-predictor-558d6f8bf8-mzwqn' metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-12b34-predictor-558d6f8bf8.18a6ef73f015ce4c namespace: kserve-ci-e2e-test resourceVersion: "19289" uid: 49e7a6dc-1c4e-4fe2-af7b-c96e4079be6a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19277" uid: 9ab84f2a-0140-44fa-9d1e-3dcf4ca08089 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Scaled up replica set error-404-isvc-12b34-predictor-558d6f8bf8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-12b34-predictor.18a6ef73eea4a9d8 namespace: kserve-ci-e2e-test resourceVersion: "19282" uid: 771f3275-15ff-4f04-ba09-dad96617ef89 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19280" uid: 4b992b88-df65-452f-a85a-3d814858c156 kind: Event lastTimestamp: "2026-04-16T20:13:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:19Z" name: error-404-isvc-12b34-predictor.18a6ef776d530715 namespace: kserve-ci-e2e-test resourceVersion: "19471" uid: f16141d3-fc6d-498c-964c-c546b8ae855d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19280" uid: 4b992b88-df65-452f-a85a-3d814858c156 kind: Event lastTimestamp: "2026-04-16T20:13:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:19Z" name: error-404-isvc-12b34-predictor.18a6ef776debd47e namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: 41c3b42a-56e3-4daf-be5f-a2877d097620 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19472" uid: 4b992b88-df65-452f-a85a-3d814858c156 kind: Event lastTimestamp: "2026-04-16T20:14:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:34Z" name: error-404-isvc-12b34-predictor.18a6ef7aec943e18 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 955691ac-9d6c-4fa9-9308-dc8401a9612c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19472" uid: 4b992b88-df65-452f-a85a-3d814858c156 kind: Event lastTimestamp: "2026-04-16T20:14:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:34Z" name: error-404-isvc-12b34-predictor.18a6ef7aecf64d97 namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: ff877d9f-7856-42bb-b7d3-498a14b56ea0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12b34 namespace: kserve-ci-e2e-test resourceVersion: "19251" uid: 17156227-d004-42a3-8253-92c239abf0a5 kind: Event lastTimestamp: "2026-04-16T20:13:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:56Z" name: error-404-isvc-12b34.18a6ef73eeacf4c7 namespace: kserve-ci-e2e-test resourceVersion: "19962" uid: dbe7b9a4-3420-4579-b702-d259ccb95464 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12b34 namespace: kserve-ci-e2e-test resourceVersion: "19251" uid: 17156227-d004-42a3-8253-92c239abf0a5 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-12b34": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-12b34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-12b34.18a6ef73ef26dfbe namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: f563e958-1710-4e6f-b96b-4946eda6e103 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12b34 namespace: kserve-ci-e2e-test resourceVersion: "19251" uid: 17156227-d004-42a3-8253-92c239abf0a5 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-12b34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-12b34.18a6ef73ef27400a namespace: kserve-ci-e2e-test resourceVersion: "19287" uid: 2ccb5e09-9b25-4154-806a-174aa920e16e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12b34 namespace: kserve-ci-e2e-test resourceVersion: "19957" uid: 17156227-d004-42a3-8253-92c239abf0a5 kind: Event lastTimestamp: "2026-04-16T20:13:55Z" message: InferenceService [error-404-isvc-12b34] is Ready metadata: creationTimestamp: "2026-04-16T20:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:55Z" name: error-404-isvc-12b34.18a6ef7ff9f2fd00 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 1b1dddf1-9058-4801-b902-8bb30d880d7f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:00:44.703582Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr namespace: kserve-ci-e2e-test resourceVersion: "12873" uid: 11d79e04-a4b3-4879-93e8-8b8729614e6f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1bf21-predictor-6b9945bd54-cggbr to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:00:44Z" name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr.18a6eec7be583625 namespace: kserve-ci-e2e-test resourceVersion: "12882" uid: 66d05819-ba07-447a-9efa-a5641e6f4746 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr namespace: kserve-ci-e2e-test resourceVersion: "12896" uid: 11d79e04-a4b3-4879-93e8-8b8729614e6f kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:00:44Z" name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr.18a6eec7c8b0bbce namespace: kserve-ci-e2e-test resourceVersion: "12923" uid: 69decd4b-2050-4c44-be27-0f8cf3ae36e8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: 11d79e04-a4b3-4879-93e8-8b8729614e6f kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:44Z" name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr.18a6eec7ca0f957a namespace: kserve-ci-e2e-test resourceVersion: "12927" uid: 733e3000-efe6-4cae-8169-da3c86bd5cec reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: 11d79e04-a4b3-4879-93e8-8b8729614e6f kind: Event lastTimestamp: "2026-04-16T20:00:59Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" in 14.312s (14.312s including waiting). Image size: 1332292201 bytes.' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:59Z" name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr.18a6eecb1f2ed3a0 namespace: kserve-ci-e2e-test resourceVersion: "13105" uid: ac3deb62-0765-4dad-926e-6220376444c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: 11d79e04-a4b3-4879-93e8-8b8729614e6f kind: Event lastTimestamp: "2026-04-16T20:00:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:59Z" name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr.18a6eecb274eb2ac namespace: kserve-ci-e2e-test resourceVersion: "13109" uid: 0db5d343-4ec2-458e-a12f-ef663f4ab470 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: 11d79e04-a4b3-4879-93e8-8b8729614e6f kind: Event lastTimestamp: "2026-04-16T20:00:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:59Z" name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr.18a6eecb27c3c78c namespace: kserve-ci-e2e-test resourceVersion: "13111" uid: e5dfd820-91e0-4291-ad96-631a7ad8e0b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: 11d79e04-a4b3-4879-93e8-8b8729614e6f kind: Event lastTimestamp: "2026-04-16T20:02:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:20Z" name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr.18a6eecb2a61607d namespace: kserve-ci-e2e-test resourceVersion: "13982" uid: eaa78733-fd56-4a13-9f62-b2ebfa76901d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr namespace: kserve-ci-e2e-test resourceVersion: "12877" uid: 11d79e04-a4b3-4879-93e8-8b8729614e6f kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:18Z" name: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr.18a6eeddac4722e7 namespace: kserve-ci-e2e-test resourceVersion: "13941" uid: 2d8023bf-099f-406c-adc3-4d4806a0b13b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1bf21-predictor-6b9945bd54 namespace: kserve-ci-e2e-test resourceVersion: "12861" uid: e3159837-7cf8-4d4b-b644-1be2fc208250 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: 'Created pod: error-404-isvc-1bf21-predictor-6b9945bd54-cggbr' metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: error-404-isvc-1bf21-predictor-6b9945bd54.18a6eec7bdc6eb06 namespace: kserve-ci-e2e-test resourceVersion: "12876" uid: 7e76bf7b-aea3-4b2f-abaf-1e00fdcf04b3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "12859" uid: 5fae7f3f-ea33-4fb9-ac6e-74ff8bcdd971 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: Scaled up replica set error-404-isvc-1bf21-predictor-6b9945bd54 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: error-404-isvc-1bf21-predictor.18a6eec7ba0b7e6b namespace: kserve-ci-e2e-test resourceVersion: "12865" uid: 4de77323-eae0-4cfd-87e4-8f4f07f85cb6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "12862" uid: aab63a49-a34d-453e-8fcb-7e562ea57023 kind: Event lastTimestamp: "2026-04-16T20:01:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:29Z" name: error-404-isvc-1bf21-predictor.18a6eecb395a53d8 namespace: kserve-ci-e2e-test resourceVersion: "13384" uid: 10ff9b88-1b73-4de4-a074-c082be31a524 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "12862" uid: aab63a49-a34d-453e-8fcb-7e562ea57023 kind: Event lastTimestamp: "2026-04-16T20:01:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:29Z" name: error-404-isvc-1bf21-predictor.18a6eecb39c4fdf9 namespace: kserve-ci-e2e-test resourceVersion: "13385" uid: 8581a6d8-4f0c-4e37-9edd-8abc171c819d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:01:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "13129" uid: aab63a49-a34d-453e-8fcb-7e562ea57023 kind: Event lastTimestamp: "2026-04-16T20:02:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:14Z" name: error-404-isvc-1bf21-predictor.18a6eed5b5c43b5d namespace: kserve-ci-e2e-test resourceVersion: "13804" uid: a3f1b1fd-4717-426f-bccf-2c84d07270c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:01:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "13129" uid: aab63a49-a34d-453e-8fcb-7e562ea57023 kind: Event lastTimestamp: "2026-04-16T20:02:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:14Z" name: error-404-isvc-1bf21-predictor.18a6eed5b635b716 namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: d7a2b3a0-9080-4d3e-99d3-481a3058ea0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf21 namespace: kserve-ci-e2e-test resourceVersion: "12845" uid: 11bcb4f0-5478-40c4-9537-c160fab1e78d kind: Event lastTimestamp: "2026-04-16T20:01:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:01:50Z" name: error-404-isvc-1bf21.18a6eec7ba69d1c6 namespace: kserve-ci-e2e-test resourceVersion: "13569" uid: a0f7e499-0f48-442e-9051-ce5001596b8e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf21 namespace: kserve-ci-e2e-test resourceVersion: "13566" uid: 11bcb4f0-5478-40c4-9537-c160fab1e78d kind: Event lastTimestamp: "2026-04-16T20:01:50Z" message: InferenceService [error-404-isvc-1bf21] is Ready metadata: creationTimestamp: "2026-04-16T20:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:01:50Z" name: error-404-isvc-1bf21.18a6eed7110c3fa0 namespace: kserve-ci-e2e-test resourceVersion: "13567" uid: 25dac5f9-7323-4e6c-ab7d-5d5d0860e45a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:13:39.904879Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: b615d820-99ec-4104-b645-5a38c0747077 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:13:39Z" name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt.18a6ef7c3bf12e6a namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 9a492499-8633-4408-9d80-ad5dce63b3a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt namespace: kserve-ci-e2e-test resourceVersion: "19783" uid: b615d820-99ec-4104-b645-5a38c0747077 kind: Event lastTimestamp: "2026-04-16T20:13:40Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:13:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:13:40Z" name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt.18a6ef7c46362db9 namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 2518d571-61ad-4cd0-9863-4a07138633c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: b615d820-99ec-4104-b645-5a38c0747077 kind: Event lastTimestamp: "2026-04-16T20:13:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:13:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:40Z" name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt.18a6ef7c48b5b2ac namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: c4410cc2-c41e-4f95-981f-23aba61ef72e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: b615d820-99ec-4104-b645-5a38c0747077 kind: Event lastTimestamp: "2026-04-16T20:13:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:13:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:40Z" name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt.18a6ef7c4fe0cd5d namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: 9651482e-30cc-455d-8f7e-34d4eb6a83e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: b615d820-99ec-4104-b645-5a38c0747077 kind: Event lastTimestamp: "2026-04-16T20:13:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:40Z" name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt.18a6ef7c5056da53 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: b67c5312-a92b-4ff5-8f18-f5891b38060b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:13:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: b615d820-99ec-4104-b645-5a38c0747077 kind: Event lastTimestamp: "2026-04-16T20:14:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:21Z" name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt.18a6ef7c7cdb56b3 namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 708fb638-617f-4e77-a51c-f569963e6613 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: b615d820-99ec-4104-b645-5a38c0747077 kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:04Z" name: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt.18a6efffbc505fdf namespace: kserve-ci-e2e-test resourceVersion: "24065" uid: d84df8e3-f6db-45e5-a43b-5a2a685394d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1f9ad-predictor-6f68d9c676 namespace: kserve-ci-e2e-test resourceVersion: "19763" uid: 776a69c1-d71e-4031-bb60-86ef9b6f188f kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: 'Created pod: error-404-isvc-1f9ad-predictor-6f68d9c676-2l6lt' metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:39Z" name: error-404-isvc-1f9ad-predictor-6f68d9c676.18a6ef7c3b8ad883 namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: e1c5ee37-407d-445c-a0bc-c6421433621e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19758" uid: 37882718-16ba-42df-bb19-88d0c212cee1 kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: Scaled up replica set error-404-isvc-1f9ad-predictor-6f68d9c676 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:39Z" name: error-404-isvc-1f9ad-predictor.18a6ef7c3a00cb29 namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: 05eaa82d-d8e4-41ab-a295-f2594a118c02 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: c65774ad-50f7-4cf4-8b9c-0df2f9181950 kind: Event lastTimestamp: "2026-04-16T20:14:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:09Z" name: error-404-isvc-1f9ad-predictor.18a6ef7fb88360aa namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 3a2f1bdc-2e0e-4a50-85cc-18448baa4129 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: c65774ad-50f7-4cf4-8b9c-0df2f9181950 kind: Event lastTimestamp: "2026-04-16T20:14:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:09Z" name: error-404-isvc-1f9ad-predictor.18a6ef7fb8e5a25a namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 5b59ee2f-0334-4b55-a1d5-f5e4ab016fce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:14:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c65774ad-50f7-4cf4-8b9c-0df2f9181950 kind: Event lastTimestamp: "2026-04-16T20:14:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:39Z" name: error-404-isvc-1f9ad-predictor.18a6ef86b677e969 namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: b7a68736-d7b2-4fee-bc4f-a681031d3921 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:14:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: c65774ad-50f7-4cf4-8b9c-0df2f9181950 kind: Event lastTimestamp: "2026-04-16T20:14:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:39Z" name: error-404-isvc-1f9ad-predictor.18a6ef86b6c36504 namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: 2c1d6ff9-ba27-4883-b89f-59c37021d2e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: 1f584821-53b2-4af1-ba8d-f99af4ef6216 kind: Event lastTimestamp: "2026-04-16T20:14:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:32Z" name: error-404-isvc-1f9ad.18a6ef7c39df86a9 namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: eb484a9e-7ba3-4df6-a6db-836523a2d081 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: 1f584821-53b2-4af1-ba8d-f99af4ef6216 kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: 'Failed to update status for InferenceService "error-404-isvc-1f9ad": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1f9ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:39Z" name: error-404-isvc-1f9ad.18a6ef7c3a47def9 namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: db5dde17-0903-4b8f-831d-efac33590143 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: 1f584821-53b2-4af1-ba8d-f99af4ef6216 kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1f9ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:39Z" name: error-404-isvc-1f9ad.18a6ef7c3a48456d namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: a0f35884-098f-45b0-a293-894fe9377f28 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 1f584821-53b2-4af1-ba8d-f99af4ef6216 kind: Event lastTimestamp: "2026-04-16T20:14:32Z" message: InferenceService [error-404-isvc-1f9ad] is Ready metadata: creationTimestamp: "2026-04-16T20:14:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:32Z" name: error-404-isvc-1f9ad.18a6ef8860e437eb namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: 7fdfba38-8eb7-46fb-8bb5-c21cc3390d31 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:02:18.887411Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2 namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: f1076105-94f3-4033-9308-97f6a7eb942b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2 to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:02:18Z" name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2.18a6eeddac235b7e namespace: kserve-ci-e2e-test resourceVersion: "13938" uid: 9c0c96b6-0553-48ba-a31c-d18f1f8f8f31 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2 namespace: kserve-ci-e2e-test resourceVersion: "13943" uid: f1076105-94f3-4033-9308-97f6a7eb942b kind: Event lastTimestamp: "2026-04-16T20:02:19Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:02:19Z" name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2.18a6eeddb5658f6b namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: 36bf25fd-33d2-47dc-93bb-bfa4dd2aa092 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2 namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: f1076105-94f3-4033-9308-97f6a7eb942b kind: Event lastTimestamp: "2026-04-16T20:02:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:19Z" name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2.18a6eeddb69b17bc namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: c0b9d0a1-9ed8-41f8-b6c0-f04039ee6336 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2 namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: f1076105-94f3-4033-9308-97f6a7eb942b kind: Event lastTimestamp: "2026-04-16T20:02:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:19Z" name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2.18a6eeddbad06c96 namespace: kserve-ci-e2e-test resourceVersion: "13958" uid: 92cd84b4-14b8-4a36-94da-8560d26b3cc7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2 namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: f1076105-94f3-4033-9308-97f6a7eb942b kind: Event lastTimestamp: "2026-04-16T20:02:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:19Z" name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2.18a6eeddbb466ed6 namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: 26e48298-2fac-4498-b9de-b791ffb8be4a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2 namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: f1076105-94f3-4033-9308-97f6a7eb942b kind: Event lastTimestamp: "2026-04-16T20:03:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:03:00Z" name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2.18a6eedddb1f03ed namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: 75b8e889-1aad-44c0-87fa-64d175846c38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2 namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: f1076105-94f3-4033-9308-97f6a7eb942b kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:44Z" name: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2.18a6ef613fa92f0b namespace: kserve-ci-e2e-test resourceVersion: "18352" uid: 114c3b62-62e0-4d7f-9026-0ef0248ff9e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3d0d2-predictor-78c799bd44 namespace: kserve-ci-e2e-test resourceVersion: "13916" uid: 67f4120b-2a44-4fdb-80cb-709aa982aa7c kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: 'Created pod: error-404-isvc-3d0d2-predictor-78c799bd44-wt5c2' metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:18Z" name: error-404-isvc-3d0d2-predictor-78c799bd44.18a6eeddabbcc38a namespace: kserve-ci-e2e-test resourceVersion: "13935" uid: aa336d89-e081-401c-92e2-bc46994d4b68 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13910" uid: 5cc30172-f13c-4213-8177-6084746ab857 kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: Scaled up replica set error-404-isvc-3d0d2-predictor-78c799bd44 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:18Z" name: error-404-isvc-3d0d2-predictor.18a6eeddaa38a922 namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: adcbe079-ecae-49c6-afdd-0a594e025374 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:02:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13914" uid: f838c0e2-3651-410d-8d9f-ff805eb2a0d3 kind: Event lastTimestamp: "2026-04-16T20:03:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:03Z" name: error-404-isvc-3d0d2-predictor.18a6eee1289d632e namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: 5f7f3bc1-512d-4048-ae02-e9585c7b3cfa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:02:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13914" uid: f838c0e2-3651-410d-8d9f-ff805eb2a0d3 kind: Event lastTimestamp: "2026-04-16T20:03:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:03Z" name: error-404-isvc-3d0d2-predictor.18a6eee12902b831 namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 7657f4da-fc67-43ea-8ace-a34b75755f94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:03:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14118" uid: f838c0e2-3651-410d-8d9f-ff805eb2a0d3 kind: Event lastTimestamp: "2026-04-16T20:04:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:03Z" name: error-404-isvc-3d0d2-predictor.18a6eeeba5367b6e namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: 02b2d1af-41a3-4bfa-aad2-92607ac906c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:03:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14118" uid: f838c0e2-3651-410d-8d9f-ff805eb2a0d3 kind: Event lastTimestamp: "2026-04-16T20:04:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:03Z" name: error-404-isvc-3d0d2-predictor.18a6eeeba5ab27fc namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: 3f5f0f40-34d1-480c-9585-e65072552750 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: a98a2c4f-d0af-4f6e-8474-622bae2c8711 kind: Event lastTimestamp: "2026-04-16T20:03:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:03:10Z" name: error-404-isvc-3d0d2.18a6eeddaa122871 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 48f57933-8492-40b4-855e-5b8a92a0b038 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: a98a2c4f-d0af-4f6e-8474-622bae2c8711 kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: 'Failed to update status for InferenceService "error-404-isvc-3d0d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3d0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:18Z" name: error-404-isvc-3d0d2.18a6eeddaad682de namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: 3c3a585f-54fe-4c59-8946-a4cee2be78d9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: a98a2c4f-d0af-4f6e-8474-622bae2c8711 kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3d0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:18Z" name: error-404-isvc-3d0d2.18a6eeddaad6cfdd namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: dcfe6d26-5588-451c-b65b-566932cd8fca reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: a98a2c4f-d0af-4f6e-8474-622bae2c8711 kind: Event lastTimestamp: "2026-04-16T20:03:10Z" message: InferenceService [error-404-isvc-3d0d2] is Ready metadata: creationTimestamp: "2026-04-16T20:03:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:03:10Z" name: error-404-isvc-3d0d2.18a6eee9c1d7d72d namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: 0533c769-e34c-4537-a6b9-70540a351296 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:23:04.849738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t namespace: kserve-ci-e2e-test resourceVersion: "24118" uid: 6793b1e0-04e8-4184-8df4-4a3694e50d66 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-653df-predictor-fd849f8f8-w5j6t to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:23:04Z" name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t.18a6efffc547a355 namespace: kserve-ci-e2e-test resourceVersion: "24122" uid: 68cfa6c7-bf04-4315-bc20-967a447b0fec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t namespace: kserve-ci-e2e-test resourceVersion: "24130" uid: 6793b1e0-04e8-4184-8df4-4a3694e50d66 kind: Event lastTimestamp: "2026-04-16T20:23:05Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:23:05Z" name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t.18a6efffcef40497 namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: 8766f639-1217-41a0-806b-2ea531bd3fba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 6793b1e0-04e8-4184-8df4-4a3694e50d66 kind: Event lastTimestamp: "2026-04-16T20:23:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:05Z" name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t.18a6efffd03bc953 namespace: kserve-ci-e2e-test resourceVersion: "24140" uid: b177a4ad-eb28-40c4-9e12-ccf1735cdcc7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 6793b1e0-04e8-4184-8df4-4a3694e50d66 kind: Event lastTimestamp: "2026-04-16T20:23:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:05Z" name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t.18a6efffd6491320 namespace: kserve-ci-e2e-test resourceVersion: "24143" uid: 38dbffa7-f9ef-4f4e-8f5a-991cb8758272 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 6793b1e0-04e8-4184-8df4-4a3694e50d66 kind: Event lastTimestamp: "2026-04-16T20:23:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:05Z" name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t.18a6efffd6c25a1c namespace: kserve-ci-e2e-test resourceVersion: "24145" uid: 1b934eee-e7d9-453e-9e6f-465a9bc2250c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 6793b1e0-04e8-4184-8df4-4a3694e50d66 kind: Event lastTimestamp: "2026-04-16T20:40:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:40:36Z" name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t.18a6f00000a5b13d namespace: kserve-ci-e2e-test resourceVersion: "32263" uid: c0f375c6-3f30-45b4-a1fa-ce7e003fd7f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 6793b1e0-04e8-4184-8df4-4a3694e50d66 kind: Event lastTimestamp: "2026-04-16T20:40:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:40:34Z" name: error-404-isvc-653df-predictor-fd849f8f8-w5j6t.18a6f0f4261c49a9 namespace: kserve-ci-e2e-test resourceVersion: "32213" uid: 735c47d1-6c91-4078-bf94-06bcd9d3dc6e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-653df-predictor-fd849f8f8 namespace: kserve-ci-e2e-test resourceVersion: "24110" uid: d38aea8f-a32d-4cc0-93c9-ee165bd6336d kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: 'Created pod: error-404-isvc-653df-predictor-fd849f8f8-w5j6t' metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:04Z" name: error-404-isvc-653df-predictor-fd849f8f8.18a6efffc4e34208 namespace: kserve-ci-e2e-test resourceVersion: "24119" uid: e61835d9-3106-4422-a734-cd16834ec366 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24108" uid: 99b8b687-d20f-48ad-8bf5-cfba51284c94 kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: Scaled up replica set error-404-isvc-653df-predictor-fd849f8f8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:04Z" name: error-404-isvc-653df-predictor.18a6efffc3855bb1 namespace: kserve-ci-e2e-test resourceVersion: "24112" uid: 792e7073-a7c6-4eaf-890b-0fb8a41dc178 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:23:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24109" uid: 6f92f41f-7fd1-4ffc-99cc-655c75fa942e kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-653df-predictor.18a6f00341f9e34f namespace: kserve-ci-e2e-test resourceVersion: "24689" uid: c7ddd360-fa04-4d54-9e89-ed4489db2097 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:23:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24109" uid: 6f92f41f-7fd1-4ffc-99cc-655c75fa942e kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-653df-predictor.18a6f0034267095c namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fbba74b9-fd8c-4664-9889-7f5143b2adf2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24281" uid: 6f92f41f-7fd1-4ffc-99cc-655c75fa942e kind: Event lastTimestamp: "2026-04-16T20:24:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:34Z" name: error-404-isvc-653df-predictor.18a6f00dbe7f7f0b namespace: kserve-ci-e2e-test resourceVersion: "25125" uid: 50075e72-4e4c-474e-8478-1f6ca451d08d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24281" uid: 6f92f41f-7fd1-4ffc-99cc-655c75fa942e kind: Event lastTimestamp: "2026-04-16T20:24:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:34Z" name: error-404-isvc-653df-predictor.18a6f00dbeffc042 namespace: kserve-ci-e2e-test resourceVersion: "25126" uid: eacbbdcb-c0fc-4aed-8e4a-1b9a685eb3c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-653df namespace: kserve-ci-e2e-test resourceVersion: "24101" uid: be56b2c1-2793-4a69-a826-e0b89d4fa9cd kind: Event lastTimestamp: "2026-04-16T20:23:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:56Z" name: error-404-isvc-653df.18a6efffc365ae2f namespace: kserve-ci-e2e-test resourceVersion: "24776" uid: 427c7bf5-70c9-4830-814a-6ed5e79f016c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-653df namespace: kserve-ci-e2e-test resourceVersion: "24101" uid: be56b2c1-2793-4a69-a826-e0b89d4fa9cd kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-653df": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-653df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:04Z" name: error-404-isvc-653df.18a6efffc3adaf4e namespace: kserve-ci-e2e-test resourceVersion: "24114" uid: 84b40fd0-e101-4b30-bb27-0e4f725cd6c2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-653df namespace: kserve-ci-e2e-test resourceVersion: "24101" uid: be56b2c1-2793-4a69-a826-e0b89d4fa9cd kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-653df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:04Z" name: error-404-isvc-653df.18a6efffc3ae13fe namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 637651b0-61f9-4594-9c1e-d1d4b94fbb4e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-653df namespace: kserve-ci-e2e-test resourceVersion: "24771" uid: be56b2c1-2793-4a69-a826-e0b89d4fa9cd kind: Event lastTimestamp: "2026-04-16T20:23:56Z" message: InferenceService [error-404-isvc-653df] is Ready metadata: creationTimestamp: "2026-04-16T20:23:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:56Z" name: error-404-isvc-653df.18a6f00be49300fc namespace: kserve-ci-e2e-test resourceVersion: "24772" uid: b2334bae-b92f-4e15-bfd9-7a43c7b9be39 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:12:19.653171Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2 namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 2767c4e9-3ec6-4973-8a0d-7a65a375385e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2 to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2.18a6ef698c915c54 namespace: kserve-ci-e2e-test resourceVersion: "18760" uid: 697610fb-4954-4846-be77-d50309308b69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2 namespace: kserve-ci-e2e-test resourceVersion: "18765" uid: 2767c4e9-3ec6-4973-8a0d-7a65a375385e kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2.18a6ef6997246c64 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 49f1592e-ce47-47d8-9d44-e66076c645ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2 namespace: kserve-ci-e2e-test resourceVersion: "18758" uid: 2767c4e9-3ec6-4973-8a0d-7a65a375385e kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2.18a6ef6999b75569 namespace: kserve-ci-e2e-test resourceVersion: "18780" uid: 470d9da4-e0c9-4854-a20d-2506bd52def9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2 namespace: kserve-ci-e2e-test resourceVersion: "18758" uid: 2767c4e9-3ec6-4973-8a0d-7a65a375385e kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2.18a6ef699dd9ba5a namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: fab678ef-b067-4486-b9ec-8203c0c21855 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2 namespace: kserve-ci-e2e-test resourceVersion: "18758" uid: 2767c4e9-3ec6-4973-8a0d-7a65a375385e kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2.18a6ef699e4bf062 namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: 083e1cab-f7f8-46f6-b630-dce8b26a8000 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:12:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2 namespace: kserve-ci-e2e-test resourceVersion: "18758" uid: 2767c4e9-3ec6-4973-8a0d-7a65a375385e kind: Event lastTimestamp: "2026-04-16T20:13:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:41Z" name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2.18a6ef69cb65b004 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 3d2cb1cd-ade5-46fd-9c8a-23b4a727dd1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2 namespace: kserve-ci-e2e-test resourceVersion: "18758" uid: 2767c4e9-3ec6-4973-8a0d-7a65a375385e kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:39Z" name: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2.18a6ef7c3aa364a7 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 3ef76880-f9f7-48b8-8e5a-1eef2659e259 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6b9e1-predictor-d9f68c7f6 namespace: kserve-ci-e2e-test resourceVersion: "18749" uid: 0d6d293a-91ec-49d9-bd76-bc1687a5efb9 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: 'Created pod: error-404-isvc-6b9e1-predictor-d9f68c7f6-rhbb2' metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-6b9e1-predictor-d9f68c7f6.18a6ef698c0063b2 namespace: kserve-ci-e2e-test resourceVersion: "18757" uid: 0f0e37ea-f938-408f-a5fb-38ec75e39d78 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18747" uid: 753d7ce7-f4bf-4a7a-8200-44f03064ef00 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Scaled up replica set error-404-isvc-6b9e1-predictor-d9f68c7f6 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-6b9e1-predictor.18a6ef698aaa0300 namespace: kserve-ci-e2e-test resourceVersion: "18751" uid: e36d6dfe-5887-4f3d-877a-8404f64c33e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:12:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18748" uid: a70e01f1-b0a8-44c0-8c74-501328ac2969 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-6b9e1-predictor.18a6ef6d091ba112 namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: ffe12738-4276-46b4-bccd-763e8a922eff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:12:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18748" uid: a70e01f1-b0a8-44c0-8c74-501328ac2969 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-6b9e1-predictor.18a6ef6d0977cb89 namespace: kserve-ci-e2e-test resourceVersion: "19323" uid: 54458da2-9b96-4ecb-9877-e879063e7475 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18917" uid: a70e01f1-b0a8-44c0-8c74-501328ac2969 kind: Event lastTimestamp: "2026-04-16T20:13:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:34Z" name: error-404-isvc-6b9e1-predictor.18a6ef7785823dc1 namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: f7ec97a1-f255-4135-855c-ca740cf305d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18917" uid: a70e01f1-b0a8-44c0-8c74-501328ac2969 kind: Event lastTimestamp: "2026-04-16T20:13:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:34Z" name: error-404-isvc-6b9e1-predictor.18a6ef7785eb927a namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: e530c783-61ed-4031-9fa0-8eda9ba8d1e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "18730" uid: 2398a7e3-0d7d-4969-9d0f-e7203a3253c1 kind: Event lastTimestamp: "2026-04-16T20:13:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:11Z" name: error-404-isvc-6b9e1.18a6ef698a889181 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 49be510d-8379-4b6a-bdc9-06cdb01bb500 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "18730" uid: 2398a7e3-0d7d-4969-9d0f-e7203a3253c1 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: 'Failed to update status for InferenceService "error-404-isvc-6b9e1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6b9e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-6b9e1.18a6ef698ae0ba7e namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: c41c6cd6-5680-4b70-a6bd-2cb981db0c4d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "18730" uid: 2398a7e3-0d7d-4969-9d0f-e7203a3253c1 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6b9e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-6b9e1.18a6ef698ae118c1 namespace: kserve-ci-e2e-test resourceVersion: "18754" uid: fe454b85-a061-45e4-bc55-ea8eb9781ef5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 2398a7e3-0d7d-4969-9d0f-e7203a3253c1 kind: Event lastTimestamp: "2026-04-16T20:13:11Z" message: InferenceService [error-404-isvc-6b9e1] is Ready metadata: creationTimestamp: "2026-04-16T20:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:11Z" name: error-404-isvc-6b9e1.18a6ef75b15eb095 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: f08d68ef-7ee9-403d-8b59-f3406e0ad220 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:02:54.716952Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl namespace: kserve-ci-e2e-test resourceVersion: "14420" uid: 816ba564-9c86-4b31-9a10-bd1bbad5fe9d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-99283-predictor-75c5d8c576-2d7bl to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:02:54Z" name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl.18a6eee603beb3be namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 7c403736-b418-42da-85a8-9518ae3a4d7e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 816ba564-9c86-4b31-9a10-bd1bbad5fe9d kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:02:54Z" name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl.18a6eee60d078e3f namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: a9ff7a07-b4b3-4cd7-a6c7-6c41b59642b9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl namespace: kserve-ci-e2e-test resourceVersion: "14422" uid: 816ba564-9c86-4b31-9a10-bd1bbad5fe9d kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl.18a6eee60e2af7ed namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 03b0c57d-f39a-4ce5-8f07-58763fd51f18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl namespace: kserve-ci-e2e-test resourceVersion: "14422" uid: 816ba564-9c86-4b31-9a10-bd1bbad5fe9d kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl.18a6eee6111e0512 namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: ebaf6580-a3b1-417b-84ff-b17373d8fb50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl namespace: kserve-ci-e2e-test resourceVersion: "14422" uid: 816ba564-9c86-4b31-9a10-bd1bbad5fe9d kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl.18a6eee6119e9992 namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: e893766e-47d3-496b-82bf-c459c14ca9bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:02:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl namespace: kserve-ci-e2e-test resourceVersion: "14422" uid: 816ba564-9c86-4b31-9a10-bd1bbad5fe9d kind: Event lastTimestamp: "2026-04-16T20:03:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:02:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:03:36Z" name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl.18a6eee644f26e33 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 3db7fc39-5932-4dd8-98c5-c56c60f4d70a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl namespace: kserve-ci-e2e-test resourceVersion: "14422" uid: 816ba564-9c86-4b31-9a10-bd1bbad5fe9d kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:19Z" name: error-404-isvc-99283-predictor-75c5d8c576-2d7bl.18a6ef6984934567 namespace: kserve-ci-e2e-test resourceVersion: "18710" uid: 4ad39175-47f1-405a-9bd1-0d241c658bc2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-99283-predictor-75c5d8c576 namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: 434090fa-b7f1-463f-a03a-5fb5b37b84b5 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: 'Created pod: error-404-isvc-99283-predictor-75c5d8c576-2d7bl' metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:54Z" name: error-404-isvc-99283-predictor-75c5d8c576.18a6eee602fee05e namespace: kserve-ci-e2e-test resourceVersion: "14421" uid: dd840c20-be60-41b8-bc88-ec59cfa95461 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: 0cd5f72b-d0b3-4c0f-b945-cbd165577597 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Scaled up replica set error-404-isvc-99283-predictor-75c5d8c576 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:54Z" name: error-404-isvc-99283-predictor.18a6eee6018bddd7 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: dc3c49b1-d2ba-40a2-8612-ea4e3b758de9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:03:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: bfd0acea-8e6a-4857-a6f6-238824bb98a9 kind: Event lastTimestamp: "2026-04-16T20:03:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:24Z" name: error-404-isvc-99283-predictor.18a6eee98002dc8d namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: af6981c9-368d-4b65-a3a8-9e7bf856438a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:03:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: bfd0acea-8e6a-4857-a6f6-238824bb98a9 kind: Event lastTimestamp: "2026-04-16T20:03:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:24Z" name: error-404-isvc-99283-predictor.18a6eee980473633 namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 247f5628-7595-444b-8f7e-72ed379cee0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:03:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: bfd0acea-8e6a-4857-a6f6-238824bb98a9 kind: Event lastTimestamp: "2026-04-16T20:04:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:09Z" name: error-404-isvc-99283-predictor.18a6eef07db958c0 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: deb30ff8-6388-4605-9cd9-b1b8aa573e09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:03:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: bfd0acea-8e6a-4857-a6f6-238824bb98a9 kind: Event lastTimestamp: "2026-04-16T20:04:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:09Z" name: error-404-isvc-99283-predictor.18a6eef07e2b506e namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 2dc55c32-5e63-4c9b-8add-2ce4ae232c59 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99283 namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: 36037042-b956-4582-a6ef-055e15fa1b18 kind: Event lastTimestamp: "2026-04-16T20:03:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:03:46Z" name: error-404-isvc-99283.18a6eee60165b304 namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: 83ac08d0-f7cb-4290-ae75-690e7165fe23 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99283 namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 36037042-b956-4582-a6ef-055e15fa1b18 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: 'Failed to update status for InferenceService "error-404-isvc-99283": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-99283": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:54Z" name: error-404-isvc-99283.18a6eee607201e00 namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: f68da98f-726c-4e84-b456-7d2fa60b7d46 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99283 namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 36037042-b956-4582-a6ef-055e15fa1b18 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-99283": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:54Z" name: error-404-isvc-99283.18a6eee607208f2e namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: f5c549cc-b029-447c-be54-672825929012 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99283 namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: 36037042-b956-4582-a6ef-055e15fa1b18 kind: Event lastTimestamp: "2026-04-16T20:03:46Z" message: InferenceService [error-404-isvc-99283] is Ready metadata: creationTimestamp: "2026-04-16T20:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:03:46Z" name: error-404-isvc-99283.18a6eef228f30964 namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: cb673d2f-52ec-4a39-b4aa-ae3ab36bdb10 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:23:49.488171Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff namespace: kserve-ci-e2e-test resourceVersion: "24655" uid: c34f3dfa-55e4-4781-9399-d716b78583d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9a649-predictor-779bcbc86d-k8kff to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff.18a6f00a29f01528 namespace: kserve-ci-e2e-test resourceVersion: "24659" uid: bcd40f42-3c0f-4312-9dcd-9199cff75643 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff namespace: kserve-ci-e2e-test resourceVersion: "24664" uid: c34f3dfa-55e4-4781-9399-d716b78583d0 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Add eth0 [10.132.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff.18a6f00a34c962ae namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: ba50a017-f5b6-4e5b-ab10-a2337e4ddfc4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff namespace: kserve-ci-e2e-test resourceVersion: "24656" uid: c34f3dfa-55e4-4781-9399-d716b78583d0 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff.18a6f00a35f68c9e namespace: kserve-ci-e2e-test resourceVersion: "24678" uid: ef317336-0321-4940-a728-3f1895a7e699 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff namespace: kserve-ci-e2e-test resourceVersion: "24656" uid: c34f3dfa-55e4-4781-9399-d716b78583d0 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff.18a6f00a3ea4ff9f namespace: kserve-ci-e2e-test resourceVersion: "24685" uid: 4764d719-cd17-4e73-ae25-2e6af34d96fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff namespace: kserve-ci-e2e-test resourceVersion: "24656" uid: c34f3dfa-55e4-4781-9399-d716b78583d0 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff.18a6f00a3f1e433f namespace: kserve-ci-e2e-test resourceVersion: "24687" uid: 8b0a05c3-a23e-4e11-a0e1-b9d62516d7be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff namespace: kserve-ci-e2e-test resourceVersion: "24656" uid: c34f3dfa-55e4-4781-9399-d716b78583d0 kind: Event lastTimestamp: "2026-04-16T20:24:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:24:31Z" name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff.18a6f00a486a7390 namespace: kserve-ci-e2e-test resourceVersion: "25091" uid: 0c2a2092-a433-479b-8866-45a90de57f3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff namespace: kserve-ci-e2e-test resourceVersion: "24656" uid: c34f3dfa-55e4-4781-9399-d716b78583d0 kind: Event lastTimestamp: "2026-04-16T20:33:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:33:14Z" name: error-404-isvc-9a649-predictor-779bcbc86d-k8kff.18a6f08db0d1cca2 namespace: kserve-ci-e2e-test resourceVersion: "29158" uid: f3624229-fbe5-444e-8c60-145f65d3f2dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9a649-predictor-779bcbc86d namespace: kserve-ci-e2e-test resourceVersion: "24648" uid: e6acee24-1000-454c-8707-050ff42343e6 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'Created pod: error-404-isvc-9a649-predictor-779bcbc86d-k8kff' metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-9a649-predictor-779bcbc86d.18a6f00a299ac225 namespace: kserve-ci-e2e-test resourceVersion: "24657" uid: b4d16aa4-68cd-4747-b465-513e54d2b056 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24646" uid: 25a3a263-0619-41e3-ae0d-47c27ffd0526 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Scaled up replica set error-404-isvc-9a649-predictor-779bcbc86d from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-9a649-predictor.18a6f00a286cf144 namespace: kserve-ci-e2e-test resourceVersion: "24650" uid: 723bf683-68e0-439c-9492-83237f3079cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24647" uid: 57d8f82d-27da-4759-9dfd-a26fe85226c4 kind: Event lastTimestamp: "2026-04-16T20:24:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:04Z" name: error-404-isvc-9a649-predictor.18a6f00da77efa69 namespace: kserve-ci-e2e-test resourceVersion: "24835" uid: 54da750a-9cc4-409b-a720-bce1e4046da8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24647" uid: 57d8f82d-27da-4759-9dfd-a26fe85226c4 kind: Event lastTimestamp: "2026-04-16T20:24:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:04Z" name: error-404-isvc-9a649-predictor.18a6f00da7efbd7e namespace: kserve-ci-e2e-test resourceVersion: "24837" uid: 75eb9a23-801d-41cb-929b-ec4f27502cc5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:24:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24836" uid: 57d8f82d-27da-4759-9dfd-a26fe85226c4 kind: Event lastTimestamp: "2026-04-16T20:25:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:24:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:25:04Z" name: error-404-isvc-9a649-predictor.18a6f011268cafc7 namespace: kserve-ci-e2e-test resourceVersion: "25402" uid: eca8782b-bf48-49dc-a14f-3e332eb60be8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:24:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24836" uid: 57d8f82d-27da-4759-9dfd-a26fe85226c4 kind: Event lastTimestamp: "2026-04-16T20:25:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:24:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:25:04Z" name: error-404-isvc-9a649-predictor.18a6f0112700ed46 namespace: kserve-ci-e2e-test resourceVersion: "25403" uid: 7ed22a22-3e24-4b11-8f12-92268afc442d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a649 namespace: kserve-ci-e2e-test resourceVersion: "24630" uid: c0597834-7b23-4363-851b-2bb8ba0a4378 kind: Event lastTimestamp: "2026-04-16T20:24:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:24:41Z" name: error-404-isvc-9a649.18a6f00a284b0feb namespace: kserve-ci-e2e-test resourceVersion: "25184" uid: ecdb2886-b30f-4d53-9e0d-9bb3c7e114e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a649 namespace: kserve-ci-e2e-test resourceVersion: "24630" uid: c0597834-7b23-4363-851b-2bb8ba0a4378 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'Failed to update status for InferenceService "error-404-isvc-9a649": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9a649": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-9a649.18a6f00a28b18737 namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 5ed2e14c-f445-42fa-af46-6cdc78737cdb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a649 namespace: kserve-ci-e2e-test resourceVersion: "24630" uid: c0597834-7b23-4363-851b-2bb8ba0a4378 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9a649": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-9a649.18a6f00a28b1d7eb namespace: kserve-ci-e2e-test resourceVersion: "24653" uid: 4fbb4e4c-3084-4d03-8571-4f550a564fac reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a649 namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: c0597834-7b23-4363-851b-2bb8ba0a4378 kind: Event lastTimestamp: "2026-04-16T20:24:41Z" message: InferenceService [error-404-isvc-9a649] is Ready metadata: creationTimestamp: "2026-04-16T20:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:24:41Z" name: error-404-isvc-9a649.18a6f0162d1bf432 namespace: kserve-ci-e2e-test resourceVersion: "25179" uid: b8b7d8da-6451-4d9b-ac4b-55d7c258ce74 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:11:44.030327Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k namespace: kserve-ci-e2e-test resourceVersion: "18354" uid: dacdd8d2-efd1-47c1-b6d9-4f87a369513a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-de58a-predictor-56d6b9bf76-pws6k to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:11:44Z" name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k.18a6ef614147dfab namespace: kserve-ci-e2e-test resourceVersion: "18359" uid: 19ec93e1-9695-46aa-91d7-6af5ed74735e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k namespace: kserve-ci-e2e-test resourceVersion: "18365" uid: dacdd8d2-efd1-47c1-b6d9-4f87a369513a kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:11:44Z" name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k.18a6ef614b3e7e10 namespace: kserve-ci-e2e-test resourceVersion: "18372" uid: 4f4eb54e-beb0-4268-8f81-e09d22e1adb8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k namespace: kserve-ci-e2e-test resourceVersion: "18357" uid: dacdd8d2-efd1-47c1-b6d9-4f87a369513a kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:44Z" name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k.18a6ef614cbe9ed4 namespace: kserve-ci-e2e-test resourceVersion: "18376" uid: 684762ef-1f1c-42ef-8ff4-fece5cbcdbbd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k namespace: kserve-ci-e2e-test resourceVersion: "18357" uid: dacdd8d2-efd1-47c1-b6d9-4f87a369513a kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:44Z" name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k.18a6ef61537605e6 namespace: kserve-ci-e2e-test resourceVersion: "18381" uid: 58a5a42f-48c7-4eb5-884c-7c0aab2b0bdb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k namespace: kserve-ci-e2e-test resourceVersion: "18357" uid: dacdd8d2-efd1-47c1-b6d9-4f87a369513a kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:44Z" name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k.18a6ef6153eac33f namespace: kserve-ci-e2e-test resourceVersion: "18382" uid: 92787c3e-02d1-4d00-8ae1-0e1aed3ac2f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k namespace: kserve-ci-e2e-test resourceVersion: "18357" uid: dacdd8d2-efd1-47c1-b6d9-4f87a369513a kind: Event lastTimestamp: "2026-04-16T20:13:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:05Z" name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k.18a6ef61620fac0f namespace: kserve-ci-e2e-test resourceVersion: "19343" uid: e1e65361-d554-4992-95a1-8b7e897e4fe5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k namespace: kserve-ci-e2e-test resourceVersion: "18357" uid: dacdd8d2-efd1-47c1-b6d9-4f87a369513a kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:04Z" name: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k.18a6ef73ea68af92 namespace: kserve-ci-e2e-test resourceVersion: "19248" uid: 39c94821-7637-4de1-b7b2-c11c9bb252b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-de58a-predictor-56d6b9bf76 namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 2a08c6bb-feb9-4c1d-ad17-5cf2e13f9da3 kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: 'Created pod: error-404-isvc-de58a-predictor-56d6b9bf76-pws6k' metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:44Z" name: error-404-isvc-de58a-predictor-56d6b9bf76.18a6ef61408f0da6 namespace: kserve-ci-e2e-test resourceVersion: "18356" uid: f4c51472-6cf2-4b7e-a9f8-a45cb29101f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18330" uid: c10e9615-3042-4a08-ba48-34432d57dbe5 kind: Event lastTimestamp: "2026-04-16T20:11:43Z" message: Scaled up replica set error-404-isvc-de58a-predictor-56d6b9bf76 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:43Z" name: error-404-isvc-de58a-predictor.18a6ef613e80e331 namespace: kserve-ci-e2e-test resourceVersion: "18344" uid: 1b281fca-29b8-4d0e-b4ce-abf356a4794a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18338" uid: b532c6af-0aed-42e1-b3c3-d61931ccdb9c kind: Event lastTimestamp: "2026-04-16T20:11:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:58Z" name: error-404-isvc-de58a-predictor.18a6ef64bcfda9bf namespace: kserve-ci-e2e-test resourceVersion: "18516" uid: fe683cc1-d8e2-45a7-b350-d5df58b7c12a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18338" uid: b532c6af-0aed-42e1-b3c3-d61931ccdb9c kind: Event lastTimestamp: "2026-04-16T20:11:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:58Z" name: error-404-isvc-de58a-predictor.18a6ef64bd68f839 namespace: kserve-ci-e2e-test resourceVersion: "18518" uid: a6f3faca-44b1-4979-8be7-8b76479b259a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:12:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18517" uid: b532c6af-0aed-42e1-b3c3-d61931ccdb9c kind: Event lastTimestamp: "2026-04-16T20:12:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:59Z" name: error-404-isvc-de58a-predictor.18a6ef683c19a80d namespace: kserve-ci-e2e-test resourceVersion: "19161" uid: bf237575-c381-4540-81cd-6d05a93616e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:12:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18517" uid: b532c6af-0aed-42e1-b3c3-d61931ccdb9c kind: Event lastTimestamp: "2026-04-16T20:12:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:59Z" name: error-404-isvc-de58a-predictor.18a6ef683c9d04df namespace: kserve-ci-e2e-test resourceVersion: "19162" uid: 9b6e9160-2852-4e26-91e8-945f88ace3b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de58a namespace: kserve-ci-e2e-test resourceVersion: "18318" uid: 977ad125-e798-43d9-870e-57cb6ed65701 kind: Event lastTimestamp: "2026-04-16T20:12:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:12:35Z" name: error-404-isvc-de58a.18a6ef613e477185 namespace: kserve-ci-e2e-test resourceVersion: "18944" uid: 934392e5-6fcb-408a-9ce3-2e3c69262b6b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de58a namespace: kserve-ci-e2e-test resourceVersion: "18318" uid: 977ad125-e798-43d9-870e-57cb6ed65701 kind: Event lastTimestamp: "2026-04-16T20:11:43Z" message: 'Failed to update status for InferenceService "error-404-isvc-de58a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-de58a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:11:43Z" name: error-404-isvc-de58a.18a6ef613ec5b49c namespace: kserve-ci-e2e-test resourceVersion: "18346" uid: e94ae7f4-40dd-4a05-988e-f43aad545870 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de58a namespace: kserve-ci-e2e-test resourceVersion: "18318" uid: 977ad125-e798-43d9-870e-57cb6ed65701 kind: Event lastTimestamp: "2026-04-16T20:11:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-de58a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:11:43Z" name: error-404-isvc-de58a.18a6ef613ec61a00 namespace: kserve-ci-e2e-test resourceVersion: "18350" uid: 2157192b-3691-43a1-8a36-309d019ea407 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de58a namespace: kserve-ci-e2e-test resourceVersion: "18941" uid: 977ad125-e798-43d9-870e-57cb6ed65701 kind: Event lastTimestamp: "2026-04-16T20:12:35Z" message: InferenceService [error-404-isvc-de58a] is Ready metadata: creationTimestamp: "2026-04-16T20:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:12:35Z" name: error-404-isvc-de58a.18a6ef6d484ec968 namespace: kserve-ci-e2e-test resourceVersion: "18942" uid: 7fff1135-6e31-4a50-aa7e-e6cf904b648a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:22:29.159737Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr namespace: kserve-ci-e2e-test resourceVersion: "23717" uid: 3d448a51-f3bb-4062-981e-316ed8f1f8dc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr.18a6eff775fd7e9b namespace: kserve-ci-e2e-test resourceVersion: "23725" uid: 73c86616-f702-4c91-810b-d4f479d68ceb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr namespace: kserve-ci-e2e-test resourceVersion: "23729" uid: 3d448a51-f3bb-4062-981e-316ed8f1f8dc kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr.18a6eff77f36d2ed namespace: kserve-ci-e2e-test resourceVersion: "23738" uid: e1c734f2-8b13-472c-ba6b-aa784076bba8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr namespace: kserve-ci-e2e-test resourceVersion: "23720" uid: 3d448a51-f3bb-4062-981e-316ed8f1f8dc kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr.18a6eff7809b4866 namespace: kserve-ci-e2e-test resourceVersion: "23742" uid: 2c1384df-c848-4bd6-bf23-6a084ca4d08b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr namespace: kserve-ci-e2e-test resourceVersion: "23720" uid: 3d448a51-f3bb-4062-981e-316ed8f1f8dc kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr.18a6eff786607ad1 namespace: kserve-ci-e2e-test resourceVersion: "23746" uid: fd8d7ee0-2f04-4828-996b-d9e7f8ddd759 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr namespace: kserve-ci-e2e-test resourceVersion: "23720" uid: 3d448a51-f3bb-4062-981e-316ed8f1f8dc kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr.18a6eff786dd191a namespace: kserve-ci-e2e-test resourceVersion: "23747" uid: 0ce90f22-b6c9-4175-8b51-6b708180f31e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr namespace: kserve-ci-e2e-test resourceVersion: "23720" uid: 3d448a51-f3bb-4062-981e-316ed8f1f8dc kind: Event lastTimestamp: "2026-04-16T20:23:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:50Z" name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr.18a6eff796cbcf95 namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: e326ea71-7312-4f5d-bd3f-50c4173bbb4b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr namespace: kserve-ci-e2e-test resourceVersion: "23720" uid: 3d448a51-f3bb-4062-981e-316ed8f1f8dc kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:49Z" name: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr.18a6f00a2246905e namespace: kserve-ci-e2e-test resourceVersion: "24607" uid: 46661c0d-9bc3-43ef-9fe3-060ade282d76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e8b6e-predictor-64c44c8fcd namespace: kserve-ci-e2e-test resourceVersion: "23703" uid: 3942fa3d-f61a-4c28-a54e-ea28624659e2 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: 'Created pod: error-404-isvc-e8b6e-predictor-64c44c8fcd-kllhr' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-e8b6e-predictor-64c44c8fcd.18a6eff7759e1ce9 namespace: kserve-ci-e2e-test resourceVersion: "23722" uid: bd64043b-b83c-4597-96f5-ff3d26a4de65 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23695" uid: 2de8d3c6-5dc1-4db5-9192-da3c4a7df5c6 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Scaled up replica set error-404-isvc-e8b6e-predictor-64c44c8fcd from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-e8b6e-predictor.18a6eff7742d3f0a namespace: kserve-ci-e2e-test resourceVersion: "23708" uid: f6f5982c-8636-47da-909d-67ee88b35717 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23702" uid: 2ceb53b8-f8e4-4971-b9b1-2c1154d34325 kind: Event lastTimestamp: "2026-04-16T20:22:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:22:44Z" name: error-404-isvc-e8b6e-predictor.18a6effaf2dd9463 namespace: kserve-ci-e2e-test resourceVersion: "23874" uid: 8d8d44ec-57be-4422-a99c-b4624a067ffe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23702" uid: 2ceb53b8-f8e4-4971-b9b1-2c1154d34325 kind: Event lastTimestamp: "2026-04-16T20:22:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:22:44Z" name: error-404-isvc-e8b6e-predictor.18a6effaf338e15e namespace: kserve-ci-e2e-test resourceVersion: "23875" uid: 80162546-025a-4069-b1c3-00773f81affb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23873" uid: 2ceb53b8-f8e4-4971-b9b1-2c1154d34325 kind: Event lastTimestamp: "2026-04-16T20:23:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:44Z" name: error-404-isvc-e8b6e-predictor.18a6effe71ec25ce namespace: kserve-ci-e2e-test resourceVersion: "24523" uid: 4ef47a48-d8e6-4f72-b32a-66c05373ea58 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23873" uid: 2ceb53b8-f8e4-4971-b9b1-2c1154d34325 kind: Event lastTimestamp: "2026-04-16T20:23:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:44Z" name: error-404-isvc-e8b6e-predictor.18a6effe72572cff namespace: kserve-ci-e2e-test resourceVersion: "24524" uid: 219d33d7-1455-48ec-adec-0d26358fc303 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "23682" uid: 25a78f16-f1a5-4223-9e0c-30b3595d6dc4 kind: Event lastTimestamp: "2026-04-16T20:23:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:20Z" name: error-404-isvc-e8b6e.18a6eff7743380e6 namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: a7505cf2-440d-419e-b33a-acbb0425dc13 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "23682" uid: 25a78f16-f1a5-4223-9e0c-30b3595d6dc4 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-e8b6e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e8b6e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-e8b6e.18a6eff774c3e1f6 namespace: kserve-ci-e2e-test resourceVersion: "23713" uid: 7c224be8-aad2-4819-b902-a5f54af69b7b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "23682" uid: 25a78f16-f1a5-4223-9e0c-30b3595d6dc4 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e8b6e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:22:29Z" name: error-404-isvc-e8b6e.18a6eff774c42e50 namespace: kserve-ci-e2e-test resourceVersion: "23719" uid: 8813c7b3-54c8-4db3-8a34-d36ffc247056 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "24302" uid: 25a78f16-f1a5-4223-9e0c-30b3595d6dc4 kind: Event lastTimestamp: "2026-04-16T20:23:20Z" message: InferenceService [error-404-isvc-e8b6e] is Ready metadata: creationTimestamp: "2026-04-16T20:23:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:20Z" name: error-404-isvc-e8b6e.18a6f0037ab1c0bb namespace: kserve-ci-e2e-test resourceVersion: "24303" uid: 8d55e937-f174-4813-805f-a54d304d051f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:00:44.603517Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12848" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:00:44Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eec7b86150ae namespace: kserve-ci-e2e-test resourceVersion: "12854" uid: b7162ae5-b978-4a55-900c-c285e739eebd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12866" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:00:45Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:00:45Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eec7d2fccbfa namespace: kserve-ci-e2e-test resourceVersion: "12943" uid: a7c90e2b-3b8d-4c1b-a870-31e38e97a6fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:00:45Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" metadata: creationTimestamp: "2026-04-16T20:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:45Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eec7d49c55b3 namespace: kserve-ci-e2e-test resourceVersion: "12945" uid: 2e5ba881-3e3d-4a4d-ad2d-923b9b4cdc80 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:00:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" in 5.887s (5.887s including waiting). Image size: 299671472 bytes.' metadata: creationTimestamp: "2026-04-16T20:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:50Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eec93388c0d0 namespace: kserve-ci-e2e-test resourceVersion: "13021" uid: 50ce5af1-6855-4521-92d9-ccfcfe5990d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:00:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:00:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:51Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eec93734aff1 namespace: kserve-ci-e2e-test resourceVersion: "13024" uid: 551b5fea-118c-4d25-9fab-e560b0964d68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:00:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:00:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:51Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eec937c38002 namespace: kserve-ci-e2e-test resourceVersion: "13025" uid: 74532060-1d7e-488e-bca2-207debec04ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:00:56Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" metadata: creationTimestamp: "2026-04-16T20:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:56Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eeca76b487f2 namespace: kserve-ci-e2e-test resourceVersion: "13076" uid: 61e29a18-ecaa-41f6-b85e-e53e759697b8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:01:04Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" in 8.264s (8.264s including waiting). Image size: 1560613811 bytes.' metadata: creationTimestamp: "2026-04-16T20:01:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:01:04Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eecc634c1c17 namespace: kserve-ci-e2e-test resourceVersion: "13174" uid: 570b07c2-1348-4521-9eef-3c7c66bfa373 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:01:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:01:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:01:04Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eecc664da940 namespace: kserve-ci-e2e-test resourceVersion: "13175" uid: e12c5c90-9435-46cc-b172-6895ff5b06c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:01:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:01:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:01:04Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eecc66ae8974 namespace: kserve-ci-e2e-test resourceVersion: "13176" uid: 35f4d767-712f-4303-bf0c-155f499d5223 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T20:01:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:02:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:01:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:56Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eecc91bbfe79 namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: 029fd056-e379-499a-b674-650645670c25 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj namespace: kserve-ci-e2e-test resourceVersion: "12851" uid: a77ee7fc-37ba-4669-867d-7b3660de3c9b kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj.18a6eee5fd7d4ccb namespace: kserve-ci-e2e-test resourceVersion: "14381" uid: ff64bf9a-bdfb-4cd6-9f38-5f4797e7c122 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-74d68d6bdc namespace: kserve-ci-e2e-test resourceVersion: "12836" uid: f06b062e-d1a4-4df2-a7d4-e98876dad414 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-74d68d6bdc-vzbzj' metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc.18a6eec7b7f8b7a1 namespace: kserve-ci-e2e-test resourceVersion: "12850" uid: 7477166b-25b5-4ca6-8afd-3cf8a4a98a3f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12833" uid: 6d0f22c7-3da0-48f1-9907-5d76b320abb9 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-74d68d6bdc from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: isvc-sklearn-graph-1-predictor.18a6eec7b55d40e8 namespace: kserve-ci-e2e-test resourceVersion: "12841" uid: edab6bda-9665-442e-aa54-0840278d473e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12835" uid: 5ceda10d-6fab-4c55-be50-55e615305001 kind: Event lastTimestamp: "2026-04-16T20:01:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:14Z" name: isvc-sklearn-graph-1-predictor.18a6eecb33e14f99 namespace: kserve-ci-e2e-test resourceVersion: "13258" uid: 5f1f516c-cd57-4f79-a3c9-7c84f8aafdc1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12835" uid: 5ceda10d-6fab-4c55-be50-55e615305001 kind: Event lastTimestamp: "2026-04-16T20:01:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:14Z" name: isvc-sklearn-graph-1-predictor.18a6eecb34757cd4 namespace: kserve-ci-e2e-test resourceVersion: "13259" uid: 48a12c24-6926-4a4a-85ad-822ced080776 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:01:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13125" uid: 5ceda10d-6fab-4c55-be50-55e615305001 kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:44Z" name: isvc-sklearn-graph-1-predictor.18a6eed231ccfedf namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: c154c809-7f3f-4ee2-8839-dc9dcf703638 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:01:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13125" uid: 5ceda10d-6fab-4c55-be50-55e615305001 kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:44Z" name: isvc-sklearn-graph-1-predictor.18a6eed23235525a namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: a1538d17-7f24-42ef-8ca8-18305f0926f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "12810" uid: 5b13d5a2-5fd2-48ea-9877-44a05babe52f kind: Event lastTimestamp: "2026-04-16T20:02:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:16Z" name: isvc-sklearn-graph-1.18a6eec7b5091586 namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: fd7c0c7d-2edb-4029-b1bc-021ee843f68e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13827" uid: 5b13d5a2-5fd2-48ea-9877-44a05babe52f kind: Event lastTimestamp: "2026-04-16T20:02:16Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-16T20:02:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:16Z" name: isvc-sklearn-graph-1.18a6eedd1dca2def namespace: kserve-ci-e2e-test resourceVersion: "13828" uid: 0f6ed735-8f18-431c-9f61-f8d03f7276d9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:00:44.953005Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12928" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-d76b874f9-v922p to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:00:44Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eec7cd360e3e namespace: kserve-ci-e2e-test resourceVersion: "12934" uid: c7044685-029f-4b7a-944e-2421f494279f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12938" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:00:45Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:00:45Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eec7e63c74ef namespace: kserve-ci-e2e-test resourceVersion: "12952" uid: 99c77480-da05-4a00-bf71-3fef95aaafd9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:00:45Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" metadata: creationTimestamp: "2026-04-16T20:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:45Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eec7e7997aa7 namespace: kserve-ci-e2e-test resourceVersion: "12954" uid: dc82a319-85d0-4212-9af8-4da65d99d513 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:00:49Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" in 3.787s (3.787s including waiting). Image size: 299671472 bytes.' metadata: creationTimestamp: "2026-04-16T20:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:49Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eec8c9601a44 namespace: kserve-ci-e2e-test resourceVersion: "12992" uid: d3cb079c-098d-4ad9-b178-0796fbd4e736 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:00:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:49Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eec8cd8df8d8 namespace: kserve-ci-e2e-test resourceVersion: "12994" uid: 96fd41e4-19ba-45e2-84e4-e7cf9307d7e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:00:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:49Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eec8cdf1d4ab namespace: kserve-ci-e2e-test resourceVersion: "12996" uid: dfaf3e48-324f-4ae0-b13e-2aabfed091af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:00:53Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" metadata: creationTimestamp: "2026-04-16T20:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:53Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eec9be821278 namespace: kserve-ci-e2e-test resourceVersion: "13046" uid: 883ddd52-5237-4888-8994-b7e3e6757b12 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:01:08Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" in 14.793s (14.793s including waiting). Image size: 1560613811 bytes.' metadata: creationTimestamp: "2026-04-16T20:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:01:08Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eecd30435291 namespace: kserve-ci-e2e-test resourceVersion: "13201" uid: 39cde625-300f-4e1c-ae8f-9ac4ba6573a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:01:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:01:08Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eecd3374adc9 namespace: kserve-ci-e2e-test resourceVersion: "13202" uid: 3fa38893-4ca0-4759-b951-e0a0dcc06e1d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:01:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:01:08Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eecd33fa9b8b namespace: kserve-ci-e2e-test resourceVersion: "13203" uid: cfadf832-ab61-4d5e-8799-5b7d47a23689 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T20:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:02:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:10Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eecd40fe64b1 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: c15ce847-97b2-4b21-9fed-8a2cabf36a84 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p namespace: kserve-ci-e2e-test resourceVersion: "12931" uid: 6fde0b41-7529-4d1e-8fab-c2bbff747b35 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-v922p.18a6eee605fa8e35 namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 9f22fe96-386f-4cd4-8687-ccb54cc5dfbc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-d76b874f9 namespace: kserve-ci-e2e-test resourceVersion: "12912" uid: 0475ae0f-4353-4474-8b53-324a591980db kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-d76b874f9-v922p' metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: isvc-sklearn-graph-2-predictor-d76b874f9.18a6eec7ccc32329 namespace: kserve-ci-e2e-test resourceVersion: "12929" uid: cf3626a2-09f8-4b12-a75f-dda780c5e20d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12908" uid: 86df6e14-4222-46a6-a948-5acccedc3714 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-d76b874f9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: isvc-sklearn-graph-2-predictor.18a6eec7c56d7608 namespace: kserve-ci-e2e-test resourceVersion: "12916" uid: f7b57353-067e-438e-b61d-fbb905e26f78 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12910" uid: b86d98be-ad2d-42c7-b245-97c4491a3e4f kind: Event lastTimestamp: "2026-04-16T20:01:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:44Z" name: isvc-sklearn-graph-2-predictor.18a6eecb43f4ab9d namespace: kserve-ci-e2e-test resourceVersion: "13509" uid: 9b1434ba-14da-4ffa-9f89-572f929186d0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12910" uid: b86d98be-ad2d-42c7-b245-97c4491a3e4f kind: Event lastTimestamp: "2026-04-16T20:01:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:44Z" name: isvc-sklearn-graph-2-predictor.18a6eecb448eb960 namespace: kserve-ci-e2e-test resourceVersion: "13510" uid: 220127a5-5f7d-41a4-91c2-84705fe1a000 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:01:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13136" uid: b86d98be-ad2d-42c7-b245-97c4491a3e4f kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:44Z" name: isvc-sklearn-graph-2-predictor.18a6eed93fd72e2d namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: 056c5766-0650-47e7-bc3e-5f858d07fdc5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:01:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13136" uid: b86d98be-ad2d-42c7-b245-97c4491a3e4f kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:44Z" name: isvc-sklearn-graph-2-predictor.18a6eed9403f3aae namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 64fe51fb-146a-41d9-9e4b-d2b3a4b507aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "12898" uid: 4005ece2-7e1e-436a-b7ee-838b81aeaa10 kind: Event lastTimestamp: "2026-04-16T20:02:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:21Z" name: isvc-sklearn-graph-2.18a6eec7c52a1279 namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: b1060bc1-7732-4ab3-8fed-16913c74542e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13998" uid: 4005ece2-7e1e-436a-b7ee-838b81aeaa10 kind: Event lastTimestamp: "2026-04-16T20:02:20Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-16T20:02:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:20Z" name: isvc-sklearn-graph-2.18a6eede28be9d2c namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 9d19ad7c-ba36-4050-b04e-740a41e7e34b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:00:44.809687Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12901" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-x9n2v to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:00:44Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eec7c4ab2947 namespace: kserve-ci-e2e-test resourceVersion: "12909" uid: 38cf191b-3412-4dcc-8691-2e1112d3c5f9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:00:45Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:00:45Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eec7e0a9a871 namespace: kserve-ci-e2e-test resourceVersion: "12947" uid: 7b3d50c4-f573-4d51-bd78-5caabfa52740 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:00:45Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" metadata: creationTimestamp: "2026-04-16T20:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:45Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eec7e1ed23aa namespace: kserve-ci-e2e-test resourceVersion: "12950" uid: 29d673c1-2a6c-4e53-b691-c49795745675 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:00:49Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" in 3.892s (3.892s including waiting). Image size: 299671472 bytes.' metadata: creationTimestamp: "2026-04-16T20:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:49Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eec8c9ed4e5e namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: a0733925-59f0-4f84-b96a-ccc1b1cbd805 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:00:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T20:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:49Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eec8cd995e54 namespace: kserve-ci-e2e-test resourceVersion: "12995" uid: 796f37b5-ba1b-4c2a-8f6d-2e0784fb8243 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:00:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T20:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:49Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eec8ce4aaa5d namespace: kserve-ci-e2e-test resourceVersion: "12997" uid: 1a003392-5abd-4812-af61-31000f309829 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:00:53Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T20:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:53Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eec9be975ba4 namespace: kserve-ci-e2e-test resourceVersion: "13047" uid: d59b1fd0-be30-466e-ab34-10b39c66a37d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:01:27Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 33.798s (33.798s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T20:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:01:27Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eed19d288585 namespace: kserve-ci-e2e-test resourceVersion: "13352" uid: a93c1e46-fae9-4500-8625-c257c9f92436 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:01:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:01:27Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eed1a33bb97f namespace: kserve-ci-e2e-test resourceVersion: "13354" uid: 459a1079-3232-4614-aab6-730d6b488897 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:01:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:01:27Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eed1a4acea34 namespace: kserve-ci-e2e-test resourceVersion: "13355" uid: f06d47b2-cd5a-4857-bacc-c3595d9c4d5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:18Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eed1b164584c namespace: kserve-ci-e2e-test resourceVersion: "13843" uid: e5e95848-d734-4ee6-b6e0-5bbafd74cb7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v namespace: kserve-ci-e2e-test resourceVersion: "12905" uid: 3264742a-0448-4b74-89ac-1eaae6425d2e kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: isvc-xgboost-graph-predictor-784cb989b8-x9n2v.18a6eee600aefdff namespace: kserve-ci-e2e-test resourceVersion: "14411" uid: 182293f4-0b63-43b7-b771-7dc2a5163c9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "12887" uid: fd7d9ab9-53b3-4b32-b80f-e848d78bcf26 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-x9n2v' metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a6eec7c4147821 namespace: kserve-ci-e2e-test resourceVersion: "12904" uid: 46c8c211-1c0f-444a-bd71-146e099d9d0d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12881" uid: 42402e25-7c7a-4af0-a48f-5d93f607a809 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: isvc-xgboost-graph-predictor.18a6eec7bf704834 namespace: kserve-ci-e2e-test resourceVersion: "12891" uid: 06515687-05d7-4101-8419-cba2790de926 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12886" uid: f4fdbb94-3377-4d44-a708-9fd028d93122 kind: Event lastTimestamp: "2026-04-16T20:01:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:59Z" name: isvc-xgboost-graph-predictor.18a6eecb3e2a0302 namespace: kserve-ci-e2e-test resourceVersion: "13642" uid: 2dada708-afe7-4e19-852f-2152ebb95b9e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12886" uid: f4fdbb94-3377-4d44-a708-9fd028d93122 kind: Event lastTimestamp: "2026-04-16T20:01:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:59Z" name: isvc-xgboost-graph-predictor.18a6eecb3ea5b575 namespace: kserve-ci-e2e-test resourceVersion: "13643" uid: c2627696-c74f-43a0-99e9-f364162ed3bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:02:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13132" uid: f4fdbb94-3377-4d44-a708-9fd028d93122 kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:44Z" name: isvc-xgboost-graph-predictor.18a6eedcb8a37245 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 3c6116eb-5071-444d-9867-09a5a1004a60 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:02:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13132" uid: f4fdbb94-3377-4d44-a708-9fd028d93122 kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:44Z" name: isvc-xgboost-graph-predictor.18a6eedcb9110512 namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: a8e65c1f-8720-460b-ae47-1a505530b4b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "12871" uid: 2d3dd048-1bd5-4132-bd08-56eae1bf115c kind: Event lastTimestamp: "2026-04-16T20:02:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:28Z" name: isvc-xgboost-graph.18a6eec7bf9c3a25 namespace: kserve-ci-e2e-test resourceVersion: "14066" uid: b0afec7c-e541-441f-9c4f-b039b8c8b710 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: 2d3dd048-1bd5-4132-bd08-56eae1bf115c kind: Event lastTimestamp: "2026-04-16T20:02:28Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-16T20:02:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:28Z" name: isvc-xgboost-graph.18a6eedfe966c8d9 namespace: kserve-ci-e2e-test resourceVersion: "14065" uid: b871afa3-bf07-4b7b-a2ed-ee380e99ddf3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:02:44.400783Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77968cf4b8-6522x namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: bdf86144-544d-4592-9cf9-e35b28dbaf48 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-77968cf4b8-6522x to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:02:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:02:44Z" name: model-chainer-77968cf4b8-6522x.18a6eee39cda7698 namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: b9eb7dee-d9a6-4714-8535-2cbdfea4882d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:44Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77968cf4b8-6522x namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: bdf86144-544d-4592-9cf9-e35b28dbaf48 kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:44Z" name: model-chainer-77968cf4b8-6522x.18a6eee3abf012f8 namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: 6361a9e0-2979-4051-b557-350a812bdbc1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:45Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77968cf4b8-6522x namespace: kserve-ci-e2e-test resourceVersion: "14213" uid: bdf86144-544d-4592-9cf9-e35b28dbaf48 kind: Event lastTimestamp: "2026-04-16T20:02:45Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:02:45Z" name: model-chainer-77968cf4b8-6522x.18a6eee3d9bdb2fc namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 91921aca-fd9b-437a-94f2-502edc0bde02 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-6522x namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: bdf86144-544d-4592-9cf9-e35b28dbaf48 kind: Event lastTimestamp: "2026-04-16T20:02:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:45Z" name: model-chainer-77968cf4b8-6522x.18a6eee3dad93523 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: cb12cc94-c20f-4a46-b585-fab0de147bfc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-6522x namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: bdf86144-544d-4592-9cf9-e35b28dbaf48 kind: Event lastTimestamp: "2026-04-16T20:02:45Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-16T20:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:45Z" name: model-chainer-77968cf4b8-6522x.18a6eee3ddd92b85 namespace: kserve-ci-e2e-test resourceVersion: "14248" uid: d9783a20-6fba-459c-84df-14df9b59ce74 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-6522x namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: bdf86144-544d-4592-9cf9-e35b28dbaf48 kind: Event lastTimestamp: "2026-04-16T20:02:45Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-16T20:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:45Z" name: model-chainer-77968cf4b8-6522x.18a6eee3de428043 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: d549fc83-5878-4af7-a9b5-b2aca0c1c788 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-6522x namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: bdf86144-544d-4592-9cf9-e35b28dbaf48 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: model-chainer-77968cf4b8-6522x.18a6eee5f6ac77f5 namespace: kserve-ci-e2e-test resourceVersion: "14345" uid: 2b30b383-44fd-4b21-a320-376f9fea114e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:02:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-6522x namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: bdf86144-544d-4592-9cf9-e35b28dbaf48 kind: Event lastTimestamp: "2026-04-16T20:03:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:02:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:03:21Z" name: model-chainer-77968cf4b8-6522x.18a6eee679df3683 namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 32041ba2-f811-4d72-9f2d-fa4345021389 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-77968cf4b8 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: 17ba75bf-7b29-437a-985e-8ab16c430a16 kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: 'Created pod: model-chainer-77968cf4b8-6522x' metadata: creationTimestamp: "2026-04-16T20:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:44Z" name: model-chainer-77968cf4b8.18a6eee39c6de831 namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 6991550d-b164-48b8-aab9-1ae37901bbbd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: 096abe5e-7e10-428a-b25a-94c403f6bb5b kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: Scaled up replica set model-chainer-77968cf4b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:44Z" name: model-chainer.18a6eee39b674abc namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 396eed65-e6d0-440e-b38b-235c7c982fcf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: e1b9a729-aeab-4990-b92a-625d14ee38f4 kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-16T20:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:44Z" name: model-chainer.18a6eee3a0ce7064 namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: fd8aca77-de67-4f10-9667-4de471eae4a5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:14:49.920490Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1f9ad-5789fb8557-kcq6v namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 6ff19e48-ce4e-4903-9ab5-5a820885deaa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-1f9ad-5789fb8557-kcq6v to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:14:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:14:49Z" name: sequence-graph-1f9ad-5789fb8557-kcq6v.18a6ef8c8932aecf namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 9edc6952-cf9c-40b6-ac89-5494ff88261a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1f9ad-5789fb8557-kcq6v namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 6ff19e48-ce4e-4903-9ab5-5a820885deaa kind: Event lastTimestamp: "2026-04-16T20:14:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-1f9ad-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:50Z" name: sequence-graph-1f9ad-5789fb8557-kcq6v.18a6ef8c8fedb3f3 namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: b7efdca3-021e-4162-8dad-9912cee95cd0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1f9ad-5789fb8557-kcq6v namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 6ff19e48-ce4e-4903-9ab5-5a820885deaa kind: Event lastTimestamp: "2026-04-16T20:14:50Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:14:50Z" name: sequence-graph-1f9ad-5789fb8557-kcq6v.18a6ef8cc622c364 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: cb71221d-784a-41c1-9783-d4097d6b644f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f9ad} kind: Pod name: sequence-graph-1f9ad-5789fb8557-kcq6v namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 6ff19e48-ce4e-4903-9ab5-5a820885deaa kind: Event lastTimestamp: "2026-04-16T20:14:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:50Z" name: sequence-graph-1f9ad-5789fb8557-kcq6v.18a6ef8cc74e3130 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: adc043f3-da0f-43d6-a2fa-e7ce810d7d1e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f9ad} kind: Pod name: sequence-graph-1f9ad-5789fb8557-kcq6v namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 6ff19e48-ce4e-4903-9ab5-5a820885deaa kind: Event lastTimestamp: "2026-04-16T20:14:51Z" message: 'Created container: sequence-graph-1f9ad' metadata: creationTimestamp: "2026-04-16T20:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:51Z" name: sequence-graph-1f9ad-5789fb8557-kcq6v.18a6ef8cca0d4d09 namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: 9cbb77e7-7d87-4986-a6f2-8b031c617427 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f9ad} kind: Pod name: sequence-graph-1f9ad-5789fb8557-kcq6v namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 6ff19e48-ce4e-4903-9ab5-5a820885deaa kind: Event lastTimestamp: "2026-04-16T20:14:51Z" message: Started container sequence-graph-1f9ad metadata: creationTimestamp: "2026-04-16T20:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:51Z" name: sequence-graph-1f9ad-5789fb8557-kcq6v.18a6ef8cca79d697 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: a5a2d17a-b449-44ab-bb2a-92329c6c3dad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f9ad} kind: Pod name: sequence-graph-1f9ad-5789fb8557-kcq6v namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 6ff19e48-ce4e-4903-9ab5-5a820885deaa kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: Stopping container sequence-graph-1f9ad metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:04Z" name: sequence-graph-1f9ad-5789fb8557-kcq6v.18a6efffb7f02e20 namespace: kserve-ci-e2e-test resourceVersion: "24047" uid: ac2c6245-8567-443d-86ad-df2927e8fb45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:23:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f9ad} kind: Pod name: sequence-graph-1f9ad-5789fb8557-kcq6v namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 6ff19e48-ce4e-4903-9ab5-5a820885deaa kind: Event lastTimestamp: "2026-04-16T20:23:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:32Z" name: sequence-graph-1f9ad-5789fb8557-kcq6v.18a6f00076799934 namespace: kserve-ci-e2e-test resourceVersion: "24396" uid: 549c376c-e773-4992-aa74-445db7d648d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-1f9ad-5789fb8557 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 0bce823a-2247-4223-8884-b168b30d2767 kind: Event lastTimestamp: "2026-04-16T20:14:49Z" message: 'Created pod: sequence-graph-1f9ad-5789fb8557-kcq6v' metadata: creationTimestamp: "2026-04-16T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:49Z" name: sequence-graph-1f9ad-5789fb8557.18a6ef8c88be2788 namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 862d3772-6d5e-4d99-8a9a-0b4111d47855 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 6900175b-1c16-4ddc-9020-65e22fb4d586 kind: Event lastTimestamp: "2026-04-16T20:14:49Z" message: Scaled up replica set sequence-graph-1f9ad-5789fb8557 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:49Z" name: sequence-graph-1f9ad.18a6ef8c87c60dd9 namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: 7db7ec67-07b1-4d91-87cb-0802ec10048e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: e2f90a38-aeb4-4b07-be6f-1f46d1dade4e kind: Event lastTimestamp: "2026-04-16T20:14:49Z" message: InferenceGraph [sequence-graph-1f9ad] is Ready metadata: creationTimestamp: "2026-04-16T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:49Z" name: sequence-graph-1f9ad.18a6ef8c8cc3eece namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: a1772517-1bd3-4a74-a644-51fb36edda3f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:15:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 833a7732-6156-46b0-b454-40792643c506 kind: Event lastTimestamp: "2026-04-16T20:15:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:15:19Z" name: sequence-graph-1f9ad.18a6ef900636c9b7 namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: ee5cf4eb-81da-4a9a-91b8-90bd9e90cf4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:15:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 833a7732-6156-46b0-b454-40792643c506 kind: Event lastTimestamp: "2026-04-16T20:15:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:15:19Z" name: sequence-graph-1f9ad.18a6ef90068cd934 namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: 870e8b4b-28eb-4c26-b2d0-9b1e1fbb9708 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:13:29.613978Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6b9e1-58b69f49c8-6r5th namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: 8152a0ff-4f25-419f-95a8-b3c3967fdfcf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-6b9e1-58b69f49c8-6r5th to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:13:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:13:29Z" name: sequence-graph-6b9e1-58b69f49c8-6r5th.18a6ef79d68e8438 namespace: kserve-ci-e2e-test resourceVersion: "19571" uid: dc27289f-a057-4739-b1f7-1f3033b0805a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6b9e1-58b69f49c8-6r5th namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: 8152a0ff-4f25-419f-95a8-b3c3967fdfcf kind: Event lastTimestamp: "2026-04-16T20:13:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-6b9e1-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:13:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:29Z" name: sequence-graph-6b9e1-58b69f49c8-6r5th.18a6ef79e30d464a namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: 61b8a960-a36c-4b8c-8618-b79bd90aa871 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6b9e1-58b69f49c8-6r5th namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: 8152a0ff-4f25-419f-95a8-b3c3967fdfcf kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:13:30Z" name: sequence-graph-6b9e1-58b69f49c8-6r5th.18a6ef7a1340ff37 namespace: kserve-ci-e2e-test resourceVersion: "19592" uid: 43858adf-a4df-48eb-a7b8-2c9f7411fc9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b9e1} kind: Pod name: sequence-graph-6b9e1-58b69f49c8-6r5th namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: 8152a0ff-4f25-419f-95a8-b3c3967fdfcf kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:30Z" name: sequence-graph-6b9e1-58b69f49c8-6r5th.18a6ef7a1469a55d namespace: kserve-ci-e2e-test resourceVersion: "19594" uid: 4c03b40f-0f24-4d01-9bce-b7b7cf2d9c04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b9e1} kind: Pod name: sequence-graph-6b9e1-58b69f49c8-6r5th namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: 8152a0ff-4f25-419f-95a8-b3c3967fdfcf kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: 'Created container: sequence-graph-6b9e1' metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:30Z" name: sequence-graph-6b9e1-58b69f49c8-6r5th.18a6ef7a172db3d0 namespace: kserve-ci-e2e-test resourceVersion: "19595" uid: 44182739-5470-46c1-b871-7dc73ddc934b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b9e1} kind: Pod name: sequence-graph-6b9e1-58b69f49c8-6r5th namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: 8152a0ff-4f25-419f-95a8-b3c3967fdfcf kind: Event lastTimestamp: "2026-04-16T20:13:30Z" message: Started container sequence-graph-6b9e1 metadata: creationTimestamp: "2026-04-16T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:30Z" name: sequence-graph-6b9e1-58b69f49c8-6r5th.18a6ef7a179a1dd7 namespace: kserve-ci-e2e-test resourceVersion: "19596" uid: dfaba7d2-a8e5-49f1-bdd9-4f1611f32612 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b9e1} kind: Pod name: sequence-graph-6b9e1-58b69f49c8-6r5th namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: 8152a0ff-4f25-419f-95a8-b3c3967fdfcf kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: Stopping container sequence-graph-6b9e1 metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:39Z" name: sequence-graph-6b9e1-58b69f49c8-6r5th.18a6ef7c2efe49ed namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 98e88da1-690d-485e-b3f9-cf15c4099fa1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:13:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6b9e1} kind: Pod name: sequence-graph-6b9e1-58b69f49c8-6r5th namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: 8152a0ff-4f25-419f-95a8-b3c3967fdfcf kind: Event lastTimestamp: "2026-04-16T20:14:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:13:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:07Z" name: sequence-graph-6b9e1-58b69f49c8-6r5th.18a6ef7cdb99ea75 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 21009f41-81da-46c5-a592-4e2ce3cacf09 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-6b9e1-58b69f49c8 namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: 7e5c528b-ff5a-4912-a5b6-0b103838ecc4 kind: Event lastTimestamp: "2026-04-16T20:13:29Z" message: 'Created pod: sequence-graph-6b9e1-58b69f49c8-6r5th' metadata: creationTimestamp: "2026-04-16T20:13:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:29Z" name: sequence-graph-6b9e1-58b69f49c8.18a6ef79d5e3b678 namespace: kserve-ci-e2e-test resourceVersion: "19567" uid: 67f147d2-3001-4b72-b09b-5ae3c7f6a6de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "19560" uid: b4d32a90-e26d-4935-9ac7-99ce798c9ddf kind: Event lastTimestamp: "2026-04-16T20:13:29Z" message: Scaled up replica set sequence-graph-6b9e1-58b69f49c8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:13:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:29Z" name: sequence-graph-6b9e1.18a6ef79d5084a62 namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: 94610df7-ba81-4b2d-8189-e7fa7a5f2e80 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "19577" uid: 92fece13-1809-47c1-8bb1-705a260f0f30 kind: Event lastTimestamp: "2026-04-16T20:13:29Z" message: InferenceGraph [sequence-graph-6b9e1] is Ready metadata: creationTimestamp: "2026-04-16T20:13:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:29Z" name: sequence-graph-6b9e1.18a6ef79da2a7d39 namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: a1a97aaf-32ba-42cf-bde7-830af3497e22 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "19553" uid: 92fece13-1809-47c1-8bb1-705a260f0f30 kind: Event lastTimestamp: "2026-04-16T20:13:29Z" message: 'Failed to update status for InferenceGraph "sequence-graph-6b9e1": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-6b9e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:29Z" name: sequence-graph-6b9e1.18a6ef79dc2efdc1 namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 58567f73-2fac-4254-8864-1317691a6b15 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "19553" uid: 92fece13-1809-47c1-8bb1-705a260f0f30 kind: Event lastTimestamp: "2026-04-16T20:13:29Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-6b9e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:29Z" name: sequence-graph-6b9e1.18a6ef79dc2f55b4 namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 840fd3a1-0497-44c1-881f-303b1e342542 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:04:04.689079Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-99283-794d7b9b66-gv4p9 namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: b94a4630-744f-491c-8350-af9ca0eb5a26 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-99283-794d7b9b66-gv4p9 to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:04:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:04:04Z" name: sequence-graph-99283-794d7b9b66-gv4p9.18a6eef64e68aafb namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: 4262cbc5-5a57-4de2-adf5-e07d17e7a522 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:04Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-99283-794d7b9b66-gv4p9 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: b94a4630-744f-491c-8350-af9ca0eb5a26 kind: Event lastTimestamp: "2026-04-16T20:04:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-99283-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:04:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:04Z" name: sequence-graph-99283-794d7b9b66-gv4p9.18a6eef65f2e04c1 namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: d416963a-dd80-496c-859a-98fd6dfff7e6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:05Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-99283-794d7b9b66-gv4p9 namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: b94a4630-744f-491c-8350-af9ca0eb5a26 kind: Event lastTimestamp: "2026-04-16T20:04:05Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:04:05Z" name: sequence-graph-99283-794d7b9b66-gv4p9.18a6eef68b1bf7ef namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 91242a4e-48bc-40dc-ac4c-d1e6ff3870f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99283} kind: Pod name: sequence-graph-99283-794d7b9b66-gv4p9 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: b94a4630-744f-491c-8350-af9ca0eb5a26 kind: Event lastTimestamp: "2026-04-16T20:04:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:05Z" name: sequence-graph-99283-794d7b9b66-gv4p9.18a6eef68c55aced namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 53bf40be-eb9c-43c6-8c3e-c845c0900744 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99283} kind: Pod name: sequence-graph-99283-794d7b9b66-gv4p9 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: b94a4630-744f-491c-8350-af9ca0eb5a26 kind: Event lastTimestamp: "2026-04-16T20:04:05Z" message: 'Created container: sequence-graph-99283' metadata: creationTimestamp: "2026-04-16T20:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:05Z" name: sequence-graph-99283-794d7b9b66-gv4p9.18a6eef68f727d0b namespace: kserve-ci-e2e-test resourceVersion: "15144" uid: 58d7a080-8940-43c5-8aaa-f4b0aadb0ebe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99283} kind: Pod name: sequence-graph-99283-794d7b9b66-gv4p9 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: b94a4630-744f-491c-8350-af9ca0eb5a26 kind: Event lastTimestamp: "2026-04-16T20:04:05Z" message: Started container sequence-graph-99283 metadata: creationTimestamp: "2026-04-16T20:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:04:05Z" name: sequence-graph-99283-794d7b9b66-gv4p9.18a6eef68fe19bbf namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 0cb4e375-4fe1-4871-90a0-812f644f0999 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99283} kind: Pod name: sequence-graph-99283-794d7b9b66-gv4p9 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: b94a4630-744f-491c-8350-af9ca0eb5a26 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Stopping container sequence-graph-99283 metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:19Z" name: sequence-graph-99283-794d7b9b66-gv4p9.18a6ef697fbbe416 namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: d112b09e-4182-4476-b1ec-29d8088d9bc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99283} kind: Pod name: sequence-graph-99283-794d7b9b66-gv4p9 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: b94a4630-744f-491c-8350-af9ca0eb5a26 kind: Event lastTimestamp: "2026-04-16T20:12:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:46Z" name: sequence-graph-99283-794d7b9b66-gv4p9.18a6ef6a158739d7 namespace: kserve-ci-e2e-test resourceVersion: "19030" uid: 32f9ea9f-4e97-48d4-bf45-8783c5d07ccc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-99283-794d7b9b66 namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: c82d7896-3b31-48bb-bcb6-4511851611d0 kind: Event lastTimestamp: "2026-04-16T20:04:04Z" message: 'Created pod: sequence-graph-99283-794d7b9b66-gv4p9' metadata: creationTimestamp: "2026-04-16T20:04:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:04Z" name: sequence-graph-99283-794d7b9b66.18a6eef64dfa593b namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: 280cbf99-ddbe-4e78-94fd-1c9e3cc97a3b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-99283 namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 24836d2a-3a53-41c3-9569-9150fd8d3d44 kind: Event lastTimestamp: "2026-04-16T20:04:04Z" message: Scaled up replica set sequence-graph-99283-794d7b9b66 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:04:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:04Z" name: sequence-graph-99283.18a6eef64d44db14 namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: 88585f9e-977a-4192-bb84-1b873c33970d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:04:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-99283 namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: 11e20791-bccb-482d-8995-37b174bf17b7 kind: Event lastTimestamp: "2026-04-16T20:04:04Z" message: InferenceGraph [sequence-graph-99283] is Ready metadata: creationTimestamp: "2026-04-16T20:04:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:04:04Z" name: sequence-graph-99283.18a6eef65206a2cf namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 1ed913e0-3f21-4353-b49e-7dfbf1e14910 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:04:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-99283 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 8dfdbc3f-23d5-40b6-a538-0bec82fd5d13 kind: Event lastTimestamp: "2026-04-16T20:04:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:34Z" name: sequence-graph-99283.18a6eef9cbe3c58a namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: afa76db9-4929-4747-b1e7-57a7ab55684d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:04:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-99283 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 8dfdbc3f-23d5-40b6-a538-0bec82fd5d13 kind: Event lastTimestamp: "2026-04-16T20:04:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:04:34Z" name: sequence-graph-99283.18a6eef9cc4982be namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: a7441a9a-aa14-4c45-83dd-4d5e8ff60613 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:24:59.504543Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9a649-6c6f5d7c86-mqbhj namespace: kserve-ci-e2e-test resourceVersion: "25339" uid: 865d281a-1fd9-46f7-82fa-d4451e002661 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-9a649-6c6f5d7c86-mqbhj to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:24:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:24:59Z" name: splitter-graph-9a649-6c6f5d7c86-mqbhj.18a6f01a773d2d75 namespace: kserve-ci-e2e-test resourceVersion: "25344" uid: 64bcbe65-4f9f-4956-b0cc-59fbfc037af8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:59Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9a649-6c6f5d7c86-mqbhj namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 865d281a-1fd9-46f7-82fa-d4451e002661 kind: Event lastTimestamp: "2026-04-16T20:24:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-9a649-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:24:59Z" name: splitter-graph-9a649-6c6f5d7c86-mqbhj.18a6f01a7df9a7ef namespace: kserve-ci-e2e-test resourceVersion: "25355" uid: dd3f4c9e-dc30-43c3-868b-afa1cc73c544 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:25:00Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9a649-6c6f5d7c86-mqbhj namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: 865d281a-1fd9-46f7-82fa-d4451e002661 kind: Event lastTimestamp: "2026-04-16T20:25:00Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:25:00Z" name: splitter-graph-9a649-6c6f5d7c86-mqbhj.18a6f01ab3cef10a namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: e972e9d1-f2a3-41ee-abc2-5e93ef53e677 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9a649} kind: Pod name: splitter-graph-9a649-6c6f5d7c86-mqbhj namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 865d281a-1fd9-46f7-82fa-d4451e002661 kind: Event lastTimestamp: "2026-04-16T20:25:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:25:00Z" name: splitter-graph-9a649-6c6f5d7c86-mqbhj.18a6f01ab4fd0a61 namespace: kserve-ci-e2e-test resourceVersion: "25368" uid: a8a866fa-f3a6-4aec-9967-100eec991418 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9a649} kind: Pod name: splitter-graph-9a649-6c6f5d7c86-mqbhj namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 865d281a-1fd9-46f7-82fa-d4451e002661 kind: Event lastTimestamp: "2026-04-16T20:25:00Z" message: 'Created container: splitter-graph-9a649' metadata: creationTimestamp: "2026-04-16T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:25:00Z" name: splitter-graph-9a649-6c6f5d7c86-mqbhj.18a6f01ab7e3d778 namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 89541ef5-26be-4057-a75a-4915b828071b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9a649} kind: Pod name: splitter-graph-9a649-6c6f5d7c86-mqbhj namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 865d281a-1fd9-46f7-82fa-d4451e002661 kind: Event lastTimestamp: "2026-04-16T20:25:00Z" message: Started container splitter-graph-9a649 metadata: creationTimestamp: "2026-04-16T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:25:00Z" name: splitter-graph-9a649-6c6f5d7c86-mqbhj.18a6f01ab84ce28c namespace: kserve-ci-e2e-test resourceVersion: "25370" uid: 6cd89e8e-681f-4185-b02b-f442a0211c7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9a649} kind: Pod name: splitter-graph-9a649-6c6f5d7c86-mqbhj namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 865d281a-1fd9-46f7-82fa-d4451e002661 kind: Event lastTimestamp: "2026-04-16T20:33:14Z" message: Stopping container splitter-graph-9a649 metadata: creationTimestamp: "2026-04-16T20:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:33:14Z" name: splitter-graph-9a649-6c6f5d7c86-mqbhj.18a6f08da719f58a namespace: kserve-ci-e2e-test resourceVersion: "29132" uid: bca50fb6-9857-4a8b-9f2d-3bf2e708fc05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9a649} kind: Pod name: splitter-graph-9a649-6c6f5d7c86-mqbhj namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 865d281a-1fd9-46f7-82fa-d4451e002661 kind: Event lastTimestamp: "2026-04-16T20:33:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:33:41Z" name: splitter-graph-9a649-6c6f5d7c86-mqbhj.18a6f08e35576253 namespace: kserve-ci-e2e-test resourceVersion: "29365" uid: f4c7a35c-af9b-410c-8cc1-4a3c5f94ac69 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-9a649-6c6f5d7c86 namespace: kserve-ci-e2e-test resourceVersion: "25336" uid: 8c260889-00f0-4a9b-9298-cbf3770aada4 kind: Event lastTimestamp: "2026-04-16T20:24:59Z" message: 'Created pod: splitter-graph-9a649-6c6f5d7c86-mqbhj' metadata: creationTimestamp: "2026-04-16T20:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:59Z" name: splitter-graph-9a649-6c6f5d7c86.18a6f01a76d2b535 namespace: kserve-ci-e2e-test resourceVersion: "25340" uid: 6d9a7abd-fb19-41d8-a190-20394331fab0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-9a649 namespace: kserve-ci-e2e-test resourceVersion: "25334" uid: 0a483b11-e691-4287-bea7-50c38c1fca76 kind: Event lastTimestamp: "2026-04-16T20:24:59Z" message: Scaled up replica set splitter-graph-9a649-6c6f5d7c86 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:59Z" name: splitter-graph-9a649.18a6f01a75ff8f27 namespace: kserve-ci-e2e-test resourceVersion: "25337" uid: fe42e59d-b604-45f2-b1bf-666c43f86cc1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9a649 namespace: kserve-ci-e2e-test resourceVersion: "25351" uid: def0755d-8426-4f97-8ea6-4766e506e23f kind: Event lastTimestamp: "2026-04-16T20:24:59Z" message: InferenceGraph [splitter-graph-9a649] is Ready metadata: creationTimestamp: "2026-04-16T20:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:24:59Z" name: splitter-graph-9a649.18a6f01a7ae0da0e namespace: kserve-ci-e2e-test resourceVersion: "25353" uid: 947d5d40-8b6e-45ab-b322-97e7c2b0a0db reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:25:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-9a649 namespace: kserve-ci-e2e-test resourceVersion: "25335" uid: 5ea5b914-9373-4c57-953e-f4cf34a6b57d kind: Event lastTimestamp: "2026-04-16T20:25:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:25:44Z" name: splitter-graph-9a649.18a6f01df49b0375 namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 93f11dd6-4665-44de-bf48-1ed071a171bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:25:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-9a649 namespace: kserve-ci-e2e-test resourceVersion: "25335" uid: 5ea5b914-9373-4c57-953e-f4cf34a6b57d kind: Event lastTimestamp: "2026-04-16T20:25:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:25:44Z" name: splitter-graph-9a649.18a6f01df50e7f97 namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: c235b0cb-ebdb-44c4-98f4-6f7e9ff0fc98 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:23:39.204532Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e8b6e-5f9c98cc85-w42z2 namespace: kserve-ci-e2e-test resourceVersion: "24464" uid: cfcdf39b-e212-4898-85fa-06d2ecb7df5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-e8b6e-5f9c98cc85-w42z2 to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:23:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:23:39Z" name: splitter-graph-e8b6e-5f9c98cc85-w42z2.18a6f007c4fc3bfb namespace: kserve-ci-e2e-test resourceVersion: "24469" uid: 5ae2e33b-b93a-4933-9656-2979ed37ea1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e8b6e-5f9c98cc85-w42z2 namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: cfcdf39b-e212-4898-85fa-06d2ecb7df5a kind: Event lastTimestamp: "2026-04-16T20:23:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-e8b6e-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:39Z" name: splitter-graph-e8b6e-5f9c98cc85-w42z2.18a6f007d14e8a01 namespace: kserve-ci-e2e-test resourceVersion: "24482" uid: 501543a8-6393-40f4-93a3-2e8e46fb71ab reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:40Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e8b6e-5f9c98cc85-w42z2 namespace: kserve-ci-e2e-test resourceVersion: "24473" uid: cfcdf39b-e212-4898-85fa-06d2ecb7df5a kind: Event lastTimestamp: "2026-04-16T20:23:40Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:23:40Z" name: splitter-graph-e8b6e-5f9c98cc85-w42z2.18a6f00801c1fcbf namespace: kserve-ci-e2e-test resourceVersion: "24489" uid: 70cd8505-682d-4169-9aa0-2378642cf7a1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e8b6e} kind: Pod name: splitter-graph-e8b6e-5f9c98cc85-w42z2 namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: cfcdf39b-e212-4898-85fa-06d2ecb7df5a kind: Event lastTimestamp: "2026-04-16T20:23:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:40Z" name: splitter-graph-e8b6e-5f9c98cc85-w42z2.18a6f00802e0c087 namespace: kserve-ci-e2e-test resourceVersion: "24492" uid: 537d5757-578c-4c9a-bbfd-bf0a67d32dff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e8b6e} kind: Pod name: splitter-graph-e8b6e-5f9c98cc85-w42z2 namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: cfcdf39b-e212-4898-85fa-06d2ecb7df5a kind: Event lastTimestamp: "2026-04-16T20:23:40Z" message: 'Created container: splitter-graph-e8b6e' metadata: creationTimestamp: "2026-04-16T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:40Z" name: splitter-graph-e8b6e-5f9c98cc85-w42z2.18a6f008060be9f3 namespace: kserve-ci-e2e-test resourceVersion: "24493" uid: 016da32b-502f-468b-83d5-8465a50d5445 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e8b6e} kind: Pod name: splitter-graph-e8b6e-5f9c98cc85-w42z2 namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: cfcdf39b-e212-4898-85fa-06d2ecb7df5a kind: Event lastTimestamp: "2026-04-16T20:23:40Z" message: Started container splitter-graph-e8b6e metadata: creationTimestamp: "2026-04-16T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:40Z" name: splitter-graph-e8b6e-5f9c98cc85-w42z2.18a6f0080675bd90 namespace: kserve-ci-e2e-test resourceVersion: "24494" uid: afeeb4b0-5899-4487-9757-7e7f559c455b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e8b6e} kind: Pod name: splitter-graph-e8b6e-5f9c98cc85-w42z2 namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: cfcdf39b-e212-4898-85fa-06d2ecb7df5a kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Stopping container splitter-graph-e8b6e metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:49Z" name: splitter-graph-e8b6e-5f9c98cc85-w42z2.18a6f00a1dcb7a18 namespace: kserve-ci-e2e-test resourceVersion: "24587" uid: ac70411c-c1ae-44f8-be63-f193811cfc78 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e8b6e} kind: Pod name: splitter-graph-e8b6e-5f9c98cc85-w42z2 namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: cfcdf39b-e212-4898-85fa-06d2ecb7df5a kind: Event lastTimestamp: "2026-04-16T20:24:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:24:16Z" name: splitter-graph-e8b6e-5f9c98cc85-w42z2.18a6f00a9a2eeef5 namespace: kserve-ci-e2e-test resourceVersion: "24962" uid: a86d1847-d3af-4b51-a033-ab612dbcdc8c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-e8b6e-5f9c98cc85 namespace: kserve-ci-e2e-test resourceVersion: "24461" uid: cc2d1641-4448-4bee-a16b-a6acc22ff142 kind: Event lastTimestamp: "2026-04-16T20:23:39Z" message: 'Created pod: splitter-graph-e8b6e-5f9c98cc85-w42z2' metadata: creationTimestamp: "2026-04-16T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:39Z" name: splitter-graph-e8b6e-5f9c98cc85.18a6f007c47365df namespace: kserve-ci-e2e-test resourceVersion: "24465" uid: 9860a0e7-7043-4e11-bb7e-f3ca0c19440a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "24459" uid: 2a95e322-142c-4249-a5ad-dd2e8b575654 kind: Event lastTimestamp: "2026-04-16T20:23:39Z" message: Scaled up replica set splitter-graph-e8b6e-5f9c98cc85 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:39Z" name: splitter-graph-e8b6e.18a6f007c393d6da namespace: kserve-ci-e2e-test resourceVersion: "24462" uid: aa63d2cf-721d-4001-a90b-1eff62cba77c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "24475" uid: a5292f3c-5d36-4fbd-88cc-7da133bad9df kind: Event lastTimestamp: "2026-04-16T20:23:39Z" message: InferenceGraph [splitter-graph-e8b6e] is Ready metadata: creationTimestamp: "2026-04-16T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:39Z" name: splitter-graph-e8b6e.18a6f007c85c642b namespace: kserve-ci-e2e-test resourceVersion: "24477" uid: 4a93399b-baab-49c7-b2b8-254e8073921d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "24452" uid: a5292f3c-5d36-4fbd-88cc-7da133bad9df kind: Event lastTimestamp: "2026-04-16T20:23:39Z" message: 'Failed to update status for InferenceGraph "splitter-graph-e8b6e": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-e8b6e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:39Z" name: splitter-graph-e8b6e.18a6f007ca53744c namespace: kserve-ci-e2e-test resourceVersion: "24479" uid: bd09da10-5904-486f-a843-e8b656fb7ec0 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "24452" uid: a5292f3c-5d36-4fbd-88cc-7da133bad9df kind: Event lastTimestamp: "2026-04-16T20:23:39Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-e8b6e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:39Z" name: splitter-graph-e8b6e.18a6f007ca53cb56 namespace: kserve-ci-e2e-test resourceVersion: "24480" uid: 8c97f761-2167-4462-87b2-d077a15c6778 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:13:04.196220Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12b34-predictor-954f5b995-lqhrg namespace: kserve-ci-e2e-test resourceVersion: "19250" uid: fc73502e-f8bc-4e27-b5b5-8e91aab00243 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-12b34-predictor-954f5b995-lqhrg to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-12b34-predictor-954f5b995-lqhrg.18a6ef73eb8a56fb namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 8aab6ce4-2fa9-4452-847d-2714b954528e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12b34-predictor-954f5b995-lqhrg namespace: kserve-ci-e2e-test resourceVersion: "19270" uid: fc73502e-f8bc-4e27-b5b5-8e91aab00243 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-12b34-predictor-954f5b995-lqhrg.18a6ef73f66553bf namespace: kserve-ci-e2e-test resourceVersion: "19303" uid: 35096ed8-6f23-4f1b-a928-aabef20d3508 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12b34-predictor-954f5b995-lqhrg namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: fc73502e-f8bc-4e27-b5b5-8e91aab00243 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-12b34-predictor-954f5b995-lqhrg.18a6ef73f7b09ea2 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: c8df05ca-7df3-4ca1-86a1-64344e898db9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12b34-predictor-954f5b995-lqhrg namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: fc73502e-f8bc-4e27-b5b5-8e91aab00243 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-12b34-predictor-954f5b995-lqhrg.18a6ef740035c84e namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: e595ce75-d3ee-4c25-b45f-cd02c98b8285 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12b34-predictor-954f5b995-lqhrg namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: fc73502e-f8bc-4e27-b5b5-8e91aab00243 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-12b34-predictor-954f5b995-lqhrg.18a6ef7400c30a69 namespace: kserve-ci-e2e-test resourceVersion: "19313" uid: dc24cbae-fc72-4485-8ffc-938dc4d9b47e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12b34-predictor-954f5b995-lqhrg namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: fc73502e-f8bc-4e27-b5b5-8e91aab00243 kind: Event lastTimestamp: "2026-04-16T20:13:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:45Z" name: success-200-isvc-12b34-predictor-954f5b995-lqhrg.18a6ef741399f7c5 namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 12dafb0d-3749-4291-addb-341b2bc77776 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12b34-predictor-954f5b995-lqhrg namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: fc73502e-f8bc-4e27-b5b5-8e91aab00243 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-12b34-predictor-954f5b995-lqhrg.18a6eff7703ca41e namespace: kserve-ci-e2e-test resourceVersion: "23680" uid: 6baf56e6-8bc6-4a58-bf03-e164cf39eb05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-12b34-predictor-954f5b995 namespace: kserve-ci-e2e-test resourceVersion: "19240" uid: ec8e8e85-d36f-469e-beb3-be5e06107db5 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'Created pod: success-200-isvc-12b34-predictor-954f5b995-lqhrg' metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-12b34-predictor-954f5b995.18a6ef73eb0b6d7e namespace: kserve-ci-e2e-test resourceVersion: "19252" uid: 91868b86-c944-443a-97d0-5fc6baac23bd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: 20d20c76-543a-4d07-b758-47714057b2b9 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Scaled up replica set success-200-isvc-12b34-predictor-954f5b995 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-12b34-predictor.18a6ef73e9817e06 namespace: kserve-ci-e2e-test resourceVersion: "19243" uid: 3a55c35e-84c0-4e45-91bd-644139cb8580 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19239" uid: 3693b5f0-caf3-4998-baed-28589ef5685e kind: Event lastTimestamp: "2026-04-16T20:13:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:19Z" name: success-200-isvc-12b34-predictor.18a6ef77682ed4ad namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: b6697750-835f-496f-8bf3-20bc98f7efd2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19239" uid: 3693b5f0-caf3-4998-baed-28589ef5685e kind: Event lastTimestamp: "2026-04-16T20:13:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:19Z" name: success-200-isvc-12b34-predictor.18a6ef7768baa30b namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: ce648212-407b-4177-b0d1-66cae4e72d9a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19469" uid: 3693b5f0-caf3-4998-baed-28589ef5685e kind: Event lastTimestamp: "2026-04-16T20:14:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:34Z" name: success-200-isvc-12b34-predictor.18a6ef7ae7c894d5 namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: 4fac1604-879c-44b7-98ff-918403c0ed05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12b34-predictor namespace: kserve-ci-e2e-test resourceVersion: "19469" uid: 3693b5f0-caf3-4998-baed-28589ef5685e kind: Event lastTimestamp: "2026-04-16T20:14:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:34Z" name: success-200-isvc-12b34-predictor.18a6ef7ae82ffb3e namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: e21765ed-106b-49b5-8332-c71ba5636007 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12b34 namespace: kserve-ci-e2e-test resourceVersion: "19222" uid: b48b8441-3db7-4e89-a581-4766184372fb kind: Event lastTimestamp: "2026-04-16T20:13:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:56Z" name: success-200-isvc-12b34.18a6ef73e95a3ab3 namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: fe4c9719-9edb-429a-a46f-5a1992a574f1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12b34 namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: b48b8441-3db7-4e89-a581-4766184372fb kind: Event lastTimestamp: "2026-04-16T20:13:55Z" message: InferenceService [success-200-isvc-12b34] is Ready metadata: creationTimestamp: "2026-04-16T20:13:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:55Z" name: success-200-isvc-12b34.18a6ef7ff7f27e69 namespace: kserve-ci-e2e-test resourceVersion: "19955" uid: 9c42aa2b-3324-45bd-926f-6105982cd4bd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:00:44.409008Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2 namespace: kserve-ci-e2e-test resourceVersion: "12811" uid: 4b09c279-8214-4252-86e3-632d35882386 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1bf21-predictor-6544f99f87-pskk2 to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:00:44Z" name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2.18a6eec7acc94578 namespace: kserve-ci-e2e-test resourceVersion: "12817" uid: 8df6ac29-96c0-451c-b0be-98317064efad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2 namespace: kserve-ci-e2e-test resourceVersion: "12820" uid: 4b09c279-8214-4252-86e3-632d35882386 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:00:44Z" name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2.18a6eec7b532c05e namespace: kserve-ci-e2e-test resourceVersion: "12840" uid: 3c6a3830-67e7-4625-9442-f97fea407fa4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2 namespace: kserve-ci-e2e-test resourceVersion: "12814" uid: 4b09c279-8214-4252-86e3-632d35882386 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:44Z" name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2.18a6eec7b6e88a67 namespace: kserve-ci-e2e-test resourceVersion: "12847" uid: 3ef2b4a8-c844-4982-bbca-6e8dbf42007b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2 namespace: kserve-ci-e2e-test resourceVersion: "12814" uid: 4b09c279-8214-4252-86e3-632d35882386 kind: Event lastTimestamp: "2026-04-16T20:00:59Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" in 14.636s (14.636s including waiting). Image size: 1332300400 bytes.' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:59Z" name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2.18a6eecb1f504b47 namespace: kserve-ci-e2e-test resourceVersion: "13106" uid: 66356eeb-d49f-46c1-be22-ef61ee1409b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2 namespace: kserve-ci-e2e-test resourceVersion: "12814" uid: 4b09c279-8214-4252-86e3-632d35882386 kind: Event lastTimestamp: "2026-04-16T20:00:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:59Z" name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2.18a6eecb2364f7bc namespace: kserve-ci-e2e-test resourceVersion: "13107" uid: bf44fd63-c11d-4d1d-9bce-8075ac20c7f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2 namespace: kserve-ci-e2e-test resourceVersion: "12814" uid: 4b09c279-8214-4252-86e3-632d35882386 kind: Event lastTimestamp: "2026-04-16T20:00:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:00:59Z" name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2.18a6eecb23c44eec namespace: kserve-ci-e2e-test resourceVersion: "13108" uid: 7abe3863-1af0-4de9-af85-29c2ec969697 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2 namespace: kserve-ci-e2e-test resourceVersion: "12814" uid: 4b09c279-8214-4252-86e3-632d35882386 kind: Event lastTimestamp: "2026-04-16T20:02:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:20Z" name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2.18a6eecb2a788d36 namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 899f096a-8049-42f2-921c-c6eb98d2b798 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2 namespace: kserve-ci-e2e-test resourceVersion: "12814" uid: 4b09c279-8214-4252-86e3-632d35882386 kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:18Z" name: success-200-isvc-1bf21-predictor-6544f99f87-pskk2.18a6eedda71541fa namespace: kserve-ci-e2e-test resourceVersion: "13897" uid: de9251af-2c21-467f-af41-8c569b1e6500 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1bf21-predictor-6544f99f87 namespace: kserve-ci-e2e-test resourceVersion: "12802" uid: 67fc6b08-b116-456a-ae20-aee34befa5c2 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: 'Created pod: success-200-isvc-1bf21-predictor-6544f99f87-pskk2' metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: success-200-isvc-1bf21-predictor-6544f99f87.18a6eec7ac05abd1 namespace: kserve-ci-e2e-test resourceVersion: "12813" uid: ea3b9c4e-5bed-40f0-ad03-7dcb52e72b3e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "12797" uid: b84f4111-b66b-4646-8693-7c16f76af249 kind: Event lastTimestamp: "2026-04-16T20:00:44Z" message: Scaled up replica set success-200-isvc-1bf21-predictor-6544f99f87 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:00:44Z" name: success-200-isvc-1bf21-predictor.18a6eec7a312890f namespace: kserve-ci-e2e-test resourceVersion: "12804" uid: 875ac13d-8ac0-449c-b46d-5f0a973731e6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "12807" uid: 07fe1ccb-0249-4a23-b8ee-6a2e0bfde8c6 kind: Event lastTimestamp: "2026-04-16T20:01:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:29Z" name: success-200-isvc-1bf21-predictor.18a6eecb2a21367b namespace: kserve-ci-e2e-test resourceVersion: "13379" uid: 5184608b-af96-4f69-b77c-4e859a98afdc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:00:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "12807" uid: 07fe1ccb-0249-4a23-b8ee-6a2e0bfde8c6 kind: Event lastTimestamp: "2026-04-16T20:01:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:01:29Z" name: success-200-isvc-1bf21-predictor.18a6eecb2a87f1fb namespace: kserve-ci-e2e-test resourceVersion: "13380" uid: c95b8307-5ec9-40e4-869c-01befd891b54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:01:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 07fe1ccb-0249-4a23-b8ee-6a2e0bfde8c6 kind: Event lastTimestamp: "2026-04-16T20:02:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:14Z" name: success-200-isvc-1bf21-predictor.18a6eed5a7594932 namespace: kserve-ci-e2e-test resourceVersion: "13792" uid: 1f67c499-8bac-41b4-8592-9e14a1b8763f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:01:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 07fe1ccb-0249-4a23-b8ee-6a2e0bfde8c6 kind: Event lastTimestamp: "2026-04-16T20:02:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:14Z" name: success-200-isvc-1bf21-predictor.18a6eed5a7bb8c17 namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: d259fef4-846b-4a66-9854-72453f52da0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:00:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf21 namespace: kserve-ci-e2e-test resourceVersion: "12788" uid: d53aa57b-1737-4428-91e6-8812dff26587 kind: Event lastTimestamp: "2026-04-16T20:01:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:01:50Z" name: success-200-isvc-1bf21.18a6eec7a8d94648 namespace: kserve-ci-e2e-test resourceVersion: "13568" uid: 59b69041-8e4d-402e-9340-a23a55c78960 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:01:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf21 namespace: kserve-ci-e2e-test resourceVersion: "13563" uid: d53aa57b-1737-4428-91e6-8812dff26587 kind: Event lastTimestamp: "2026-04-16T20:01:50Z" message: InferenceService [success-200-isvc-1bf21] is Ready metadata: creationTimestamp: "2026-04-16T20:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:01:50Z" name: success-200-isvc-1bf21.18a6eed70f23f59e namespace: kserve-ci-e2e-test resourceVersion: "13564" uid: 242e25d6-1c12-4c72-ac59-707353e5cc60 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:13:39.811118Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: 57f8f750-7681-45d3-b864-ed5e9e150e2a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:13:39Z" name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn.18a6ef7c365a940f namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 3d102d8e-a269-4f4c-875c-bb59f41b37ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: 57f8f750-7681-45d3-b864-ed5e9e150e2a kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:13:39Z" name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn.18a6ef7c4070ad33 namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: be58f75b-4955-443a-ad07-88a821262096 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: 57f8f750-7681-45d3-b864-ed5e9e150e2a kind: Event lastTimestamp: "2026-04-16T20:13:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:13:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:40Z" name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn.18a6ef7c41f6de45 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: ce554299-4b8c-484d-b540-4c1fc78a253d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: 57f8f750-7681-45d3-b864-ed5e9e150e2a kind: Event lastTimestamp: "2026-04-16T20:13:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:13:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:40Z" name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn.18a6ef7c49ef23d6 namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: beec886c-cbca-422a-8bbc-ad736adb29f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: 57f8f750-7681-45d3-b864-ed5e9e150e2a kind: Event lastTimestamp: "2026-04-16T20:13:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:40Z" name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn.18a6ef7c4a68b0ec namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 5e234a20-4b34-41fc-9948-c6368fe69802 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:13:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: 57f8f750-7681-45d3-b864-ed5e9e150e2a kind: Event lastTimestamp: "2026-04-16T20:14:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:13:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:14:22Z" name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn.18a6ef7c7ccd0f29 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 0263218d-6ace-4048-a0a7-fea073a3ad94 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: 57f8f750-7681-45d3-b864-ed5e9e150e2a kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:04Z" name: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn.18a6efffbf868109 namespace: kserve-ci-e2e-test resourceVersion: "24093" uid: 8f47fd9d-d298-43c1-937e-a0bd58705d5a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1f9ad-predictor-84d585c7c namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 0e1d79f6-45b5-456c-8a50-a8e71e80a96e kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: 'Created pod: success-200-isvc-1f9ad-predictor-84d585c7c-4pvwn' metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:39Z" name: success-200-isvc-1f9ad-predictor-84d585c7c.18a6ef7c35f2b573 namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: 282e6ab6-4bef-4a3c-aedf-1b71bcfee62f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: 92964c72-34f3-44b1-ac89-75fbad3dc309 kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: Scaled up replica set success-200-isvc-1f9ad-predictor-84d585c7c from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:39Z" name: success-200-isvc-1f9ad-predictor.18a6ef7c3465ee6c namespace: kserve-ci-e2e-test resourceVersion: "19726" uid: 6b3b4b09-e752-4a47-a862-d8c36b7ebeda reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 2a9f7d1b-2b7f-4019-b6f5-e7a41801de35 kind: Event lastTimestamp: "2026-04-16T20:14:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:09Z" name: success-200-isvc-1f9ad-predictor.18a6ef7fb35aa27e namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 13904209-7cd8-4dc5-b851-ea1e84e5a5ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 2a9f7d1b-2b7f-4019-b6f5-e7a41801de35 kind: Event lastTimestamp: "2026-04-16T20:14:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:09Z" name: success-200-isvc-1f9ad-predictor.18a6ef7fb3a49b48 namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: 9b08f759-6dd1-413d-a570-a36a070a91d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:14:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: 2a9f7d1b-2b7f-4019-b6f5-e7a41801de35 kind: Event lastTimestamp: "2026-04-16T20:14:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:54Z" name: success-200-isvc-1f9ad-predictor.18a6ef86b1b56abb namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: e1a84782-3e9f-4d02-85b7-5aa3f47377d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:14:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f9ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: 2a9f7d1b-2b7f-4019-b6f5-e7a41801de35 kind: Event lastTimestamp: "2026-04-16T20:14:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:14:54Z" name: success-200-isvc-1f9ad-predictor.18a6ef86b21da61e namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: c0abf4bd-8618-4599-8c99-94e76b7b7f60 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "19705" uid: 608dd9ea-4f37-49a6-9214-e97ef4330b1b kind: Event lastTimestamp: "2026-04-16T20:14:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:32Z" name: success-200-isvc-1f9ad.18a6ef7c34593c39 namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: 3accf2a0-1fa5-4a53-99ca-de2e4ca175fc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "19705" uid: 608dd9ea-4f37-49a6-9214-e97ef4330b1b kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: 'Failed to update status for InferenceService "success-200-isvc-1f9ad": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1f9ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:39Z" name: success-200-isvc-1f9ad.18a6ef7c3522e37c namespace: kserve-ci-e2e-test resourceVersion: "19730" uid: 2691496f-2aa3-4eff-87d4-37dcb9a402fe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "19705" uid: 608dd9ea-4f37-49a6-9214-e97ef4330b1b kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1f9ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:39Z" name: success-200-isvc-1f9ad.18a6ef7c35233cdf namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: 5885f2cb-8b7a-4962-ad5c-c1bccc6221b3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:14:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f9ad namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 608dd9ea-4f37-49a6-9214-e97ef4330b1b kind: Event lastTimestamp: "2026-04-16T20:14:32Z" message: InferenceService [success-200-isvc-1f9ad] is Ready metadata: creationTimestamp: "2026-04-16T20:14:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:14:32Z" name: success-200-isvc-1f9ad.18a6ef8862a01433 namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 597d0f8f-dcd3-40dc-8d8f-07ca2fde04fa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:02:18.836167Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669 namespace: kserve-ci-e2e-test resourceVersion: "13900" uid: 979d7b3d-8bf4-413a-80e4-606aec94cc9a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3d0d2-predictor-7967c5b545-lq669 to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:02:18Z" name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669.18a6eedda9156084 namespace: kserve-ci-e2e-test resourceVersion: "13909" uid: 891f4d6b-6afc-4649-beda-9425111bcd3c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669 namespace: kserve-ci-e2e-test resourceVersion: "13921" uid: 979d7b3d-8bf4-413a-80e4-606aec94cc9a kind: Event lastTimestamp: "2026-04-16T20:02:19Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:02:19Z" name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669.18a6eeddb30ae990 namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: 7150c215-b5ba-4a9a-983b-0a186831334c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669 namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: 979d7b3d-8bf4-413a-80e4-606aec94cc9a kind: Event lastTimestamp: "2026-04-16T20:02:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:19Z" name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669.18a6eeddb4693d45 namespace: kserve-ci-e2e-test resourceVersion: "13951" uid: 0fe500ef-bded-4b54-8b5c-dfa4d7b4756e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669 namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: 979d7b3d-8bf4-413a-80e4-606aec94cc9a kind: Event lastTimestamp: "2026-04-16T20:02:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:19Z" name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669.18a6eeddbad62e66 namespace: kserve-ci-e2e-test resourceVersion: "13959" uid: 3f88b387-adcd-4d85-8ec9-e15b22ce67dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669 namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: 979d7b3d-8bf4-413a-80e4-606aec94cc9a kind: Event lastTimestamp: "2026-04-16T20:02:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:19Z" name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669.18a6eeddbb560b1e namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: f2639b82-d15f-4ad1-b610-769d4ae71dec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669 namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: 979d7b3d-8bf4-413a-80e4-606aec94cc9a kind: Event lastTimestamp: "2026-04-16T20:03:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:03:00Z" name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669.18a6eedddb28ad7a namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: bda94c06-b7be-470d-be47-7cd2260b5429 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669 namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: 979d7b3d-8bf4-413a-80e4-606aec94cc9a kind: Event lastTimestamp: "2026-04-16T20:11:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:43Z" name: success-200-isvc-3d0d2-predictor-7967c5b545-lq669.18a6ef613a89f101 namespace: kserve-ci-e2e-test resourceVersion: "18316" uid: 4c359536-7361-4e55-af70-9a26db16345c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3d0d2-predictor-7967c5b545 namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: d32ddac1-31dd-4d81-9920-04808dd8ab60 kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: 'Created pod: success-200-isvc-3d0d2-predictor-7967c5b545-lq669' metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:18Z" name: success-200-isvc-3d0d2-predictor-7967c5b545.18a6eedda8af7f25 namespace: kserve-ci-e2e-test resourceVersion: "13903" uid: b4285e29-3006-4e34-b6e2-29d764c61ddc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13880" uid: d0c8e19f-98bc-4339-88c3-78ad90437b33 kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: Scaled up replica set success-200-isvc-3d0d2-predictor-7967c5b545 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:18Z" name: success-200-isvc-3d0d2-predictor.18a6eedda60df834 namespace: kserve-ci-e2e-test resourceVersion: "13889" uid: 16ba1b43-2937-41ad-b761-ae77e44ef395 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13882" uid: 8eed76d9-3829-4489-9eaf-2bfc93badf6e kind: Event lastTimestamp: "2026-04-16T20:02:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:33Z" name: success-200-isvc-3d0d2-predictor.18a6eee1247c2c41 namespace: kserve-ci-e2e-test resourceVersion: "14113" uid: 6d4301f7-8921-493c-afe1-1041c0e00145 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13882" uid: 8eed76d9-3829-4489-9eaf-2bfc93badf6e kind: Event lastTimestamp: "2026-04-16T20:02:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:33Z" name: success-200-isvc-3d0d2-predictor.18a6eee124e26fe8 namespace: kserve-ci-e2e-test resourceVersion: "14115" uid: 07a30898-5c93-4b67-917b-f69d280cac85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:02:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: 8eed76d9-3829-4489-9eaf-2bfc93badf6e kind: Event lastTimestamp: "2026-04-16T20:03:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:02:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:33Z" name: success-200-isvc-3d0d2-predictor.18a6eee4a37ec782 namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 7dea3f0e-d277-4813-bb90-8fb96b1b26c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:02:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: 8eed76d9-3829-4489-9eaf-2bfc93badf6e kind: Event lastTimestamp: "2026-04-16T20:03:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:02:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:33Z" name: success-200-isvc-3d0d2-predictor.18a6eee4a3de8bfd namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: e64fa307-9d1e-4b8b-9df7-3580ca2832fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "13866" uid: 84087be7-fa47-4d1d-9a96-2226c1a9a736 kind: Event lastTimestamp: "2026-04-16T20:03:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:03:10Z" name: success-200-isvc-3d0d2.18a6eedda5e55a0f namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: ac17c0d2-b288-48d1-8b93-33758b7a5b2f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "13866" uid: 84087be7-fa47-4d1d-9a96-2226c1a9a736 kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: 'Failed to update status for InferenceService "success-200-isvc-3d0d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3d0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:18Z" name: success-200-isvc-3d0d2.18a6eedda679c3ba namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: cf0ee53f-ea20-4c33-a99a-39e910296c05 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "13866" uid: 84087be7-fa47-4d1d-9a96-2226c1a9a736 kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3d0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:18Z" name: success-200-isvc-3d0d2.18a6eedda67a1fac namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 53f4cdf1-87a8-4fc0-9355-eb72f27febcc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 84087be7-fa47-4d1d-9a96-2226c1a9a736 kind: Event lastTimestamp: "2026-04-16T20:03:10Z" message: InferenceService [success-200-isvc-3d0d2] is Ready metadata: creationTimestamp: "2026-04-16T20:03:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:03:10Z" name: success-200-isvc-3d0d2.18a6eee9bfe54d0c namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: a680d742-d6f4-4f26-bd24-a02bb72b9720 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:23:04.759290Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-653df-predictor-7db8685f5-75qnk namespace: kserve-ci-e2e-test resourceVersion: "24091" uid: 48d83598-87ee-44ad-b07e-45af3b6338dc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-653df-predictor-7db8685f5-75qnk to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:23:04Z" name: success-200-isvc-653df-predictor-7db8685f5-75qnk.18a6efffbfe38a1e namespace: kserve-ci-e2e-test resourceVersion: "24096" uid: ca0a906f-94f9-454c-aaf5-dd37a3acf3a4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-653df-predictor-7db8685f5-75qnk namespace: kserve-ci-e2e-test resourceVersion: "24100" uid: 48d83598-87ee-44ad-b07e-45af3b6338dc kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:23:04Z" name: success-200-isvc-653df-predictor-7db8685f5-75qnk.18a6efffc9c94f9e namespace: kserve-ci-e2e-test resourceVersion: "24133" uid: 48933ba9-faa2-44b4-96d2-a768a3b1dcc4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-653df-predictor-7db8685f5-75qnk namespace: kserve-ci-e2e-test resourceVersion: "24092" uid: 48d83598-87ee-44ad-b07e-45af3b6338dc kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:04Z" name: success-200-isvc-653df-predictor-7db8685f5-75qnk.18a6efffcb2c1a8d namespace: kserve-ci-e2e-test resourceVersion: "24135" uid: 410cbcea-5dd9-40ca-8ff4-d3458599d36a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-653df-predictor-7db8685f5-75qnk namespace: kserve-ci-e2e-test resourceVersion: "24092" uid: 48d83598-87ee-44ad-b07e-45af3b6338dc kind: Event lastTimestamp: "2026-04-16T20:23:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:05Z" name: success-200-isvc-653df-predictor-7db8685f5-75qnk.18a6efffd177674a namespace: kserve-ci-e2e-test resourceVersion: "24141" uid: c0ad6193-06fe-44dc-a507-a1eedf38107d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-653df-predictor-7db8685f5-75qnk namespace: kserve-ci-e2e-test resourceVersion: "24092" uid: 48d83598-87ee-44ad-b07e-45af3b6338dc kind: Event lastTimestamp: "2026-04-16T20:23:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:05Z" name: success-200-isvc-653df-predictor-7db8685f5-75qnk.18a6efffd1f02b61 namespace: kserve-ci-e2e-test resourceVersion: "24142" uid: 181d164c-f420-4fd0-9405-f1117ee925ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-653df-predictor-7db8685f5-75qnk namespace: kserve-ci-e2e-test resourceVersion: "24092" uid: 48d83598-87ee-44ad-b07e-45af3b6338dc kind: Event lastTimestamp: "2026-04-16T20:40:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:40:36Z" name: success-200-isvc-653df-predictor-7db8685f5-75qnk.18a6f00000939209 namespace: kserve-ci-e2e-test resourceVersion: "32262" uid: b95ef1fe-b66d-455a-a346-7ba63f2f0e85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-653df-predictor-7db8685f5-75qnk namespace: kserve-ci-e2e-test resourceVersion: "24092" uid: 48d83598-87ee-44ad-b07e-45af3b6338dc kind: Event lastTimestamp: "2026-04-16T20:40:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:40:34Z" name: success-200-isvc-653df-predictor-7db8685f5-75qnk.18a6f0f422e40c8d namespace: kserve-ci-e2e-test resourceVersion: "32200" uid: 7389170c-a4ab-44aa-a15c-719e7fb0c7b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-653df-predictor-7db8685f5 namespace: kserve-ci-e2e-test resourceVersion: "24076" uid: a2058b01-8dbd-4ad6-87f1-9cee17117bce kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: 'Created pod: success-200-isvc-653df-predictor-7db8685f5-75qnk' metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:04Z" name: success-200-isvc-653df-predictor-7db8685f5.18a6efffbf916bb7 namespace: kserve-ci-e2e-test resourceVersion: "24094" uid: f43f150e-a6db-4386-922f-f5796385d741 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24070" uid: a77ffcb4-53c6-475f-9000-1c0b39ddc2c7 kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: Scaled up replica set success-200-isvc-653df-predictor-7db8685f5 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:04Z" name: success-200-isvc-653df-predictor.18a6efffbdfa9e1f namespace: kserve-ci-e2e-test resourceVersion: "24081" uid: d53cbbf2-e14e-4a67-bd1a-5682b7327efa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:23:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24074" uid: d8f9d4d8-5214-4056-a10e-1e8166d14137 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-653df-predictor.18a6f0033c725209 namespace: kserve-ci-e2e-test resourceVersion: "24681" uid: a6c2c8e3-2f9e-432e-802d-8803b4e4be86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:23:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24074" uid: d8f9d4d8-5214-4056-a10e-1e8166d14137 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-653df-predictor.18a6f0033cd9b8ea namespace: kserve-ci-e2e-test resourceVersion: "24682" uid: 50149fff-5b44-49d4-9345-d911e2198316 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24277" uid: d8f9d4d8-5214-4056-a10e-1e8166d14137 kind: Event lastTimestamp: "2026-04-16T20:24:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:34Z" name: success-200-isvc-653df-predictor.18a6f00db8c87d76 namespace: kserve-ci-e2e-test resourceVersion: "25120" uid: 1cad5ce6-be3f-4246-b2dd-00e4a7bf4fe9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-653df-predictor namespace: kserve-ci-e2e-test resourceVersion: "24277" uid: d8f9d4d8-5214-4056-a10e-1e8166d14137 kind: Event lastTimestamp: "2026-04-16T20:24:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:34Z" name: success-200-isvc-653df-predictor.18a6f00db90db12b namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: c5a5df84-0796-47ba-a00a-ce9930ae26f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-653df namespace: kserve-ci-e2e-test resourceVersion: "24052" uid: 882252f6-96c9-478b-9a35-abdda16751f6 kind: Event lastTimestamp: "2026-04-16T20:23:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:56Z" name: success-200-isvc-653df.18a6efffbdd878b6 namespace: kserve-ci-e2e-test resourceVersion: "24777" uid: 0024448e-1f6b-45f0-a349-93e7d6788554 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-653df namespace: kserve-ci-e2e-test resourceVersion: "24052" uid: 882252f6-96c9-478b-9a35-abdda16751f6 kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-653df": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-653df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:04Z" name: success-200-isvc-653df.18a6efffbead4786 namespace: kserve-ci-e2e-test resourceVersion: "24086" uid: 7113f6ff-bc07-4712-9dbf-7b05c57a809a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-653df namespace: kserve-ci-e2e-test resourceVersion: "24052" uid: 882252f6-96c9-478b-9a35-abdda16751f6 kind: Event lastTimestamp: "2026-04-16T20:23:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-653df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:04Z" name: success-200-isvc-653df.18a6efffbeadbeb0 namespace: kserve-ci-e2e-test resourceVersion: "24089" uid: 6fd571bb-c988-4707-93c6-6bf8d82c0aae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-653df namespace: kserve-ci-e2e-test resourceVersion: "24774" uid: 882252f6-96c9-478b-9a35-abdda16751f6 kind: Event lastTimestamp: "2026-04-16T20:23:56Z" message: InferenceService [success-200-isvc-653df] is Ready metadata: creationTimestamp: "2026-04-16T20:23:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:56Z" name: success-200-isvc-653df.18a6f00be63f61f6 namespace: kserve-ci-e2e-test resourceVersion: "24775" uid: 85e25247-8535-435c-b6da-3e36d1556fc4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:12:19.576874Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr namespace: kserve-ci-e2e-test resourceVersion: "18729" uid: 824c77c0-f514-44fa-9c4a-fe78e3de0d02 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:12:19Z" name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr.18a6ef6988052ce3 namespace: kserve-ci-e2e-test resourceVersion: "18736" uid: c39e81d0-3d83-4dce-b7b0-3413126ccf97 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: 824c77c0-f514-44fa-9c4a-fe78e3de0d02 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:12:19Z" name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr.18a6ef6990d7937f namespace: kserve-ci-e2e-test resourceVersion: "18769" uid: 38572f50-d987-43a6-92bb-b19ba67989f7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 824c77c0-f514-44fa-9c4a-fe78e3de0d02 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:19Z" name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr.18a6ef69921da74e namespace: kserve-ci-e2e-test resourceVersion: "18772" uid: 97273ecc-6ea9-4e04-888e-cc18f7e90087 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 824c77c0-f514-44fa-9c4a-fe78e3de0d02 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:19Z" name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr.18a6ef6997a1677a namespace: kserve-ci-e2e-test resourceVersion: "18777" uid: 3105fc3a-c8b9-4c7c-918a-d86dc6d07530 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 824c77c0-f514-44fa-9c4a-fe78e3de0d02 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:19Z" name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr.18a6ef6998200a01 namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 8cd3eacc-1278-4e3f-b39f-8fc2234b47b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:12:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 824c77c0-f514-44fa-9c4a-fe78e3de0d02 kind: Event lastTimestamp: "2026-04-16T20:13:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:41Z" name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr.18a6ef69cb7135a9 namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: 4de11ec6-1443-4e56-b8da-4579c5deca3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 824c77c0-f514-44fa-9c4a-fe78e3de0d02 kind: Event lastTimestamp: "2026-04-16T20:13:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:39Z" name: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr.18a6ef7c359700cb namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 1ba79630-ed04-438c-87db-dee14435356c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6b9e1-predictor-77fcc484d9 namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: a82b7f9a-0c1b-43a2-8cd4-f65e406c5a9c kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: 'Created pod: success-200-isvc-6b9e1-predictor-77fcc484d9-s29lr' metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:19Z" name: success-200-isvc-6b9e1-predictor-77fcc484d9.18a6ef6987901c22 namespace: kserve-ci-e2e-test resourceVersion: "18731" uid: ed750638-7b40-4060-8b3e-082ccbb52ee0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18709" uid: b914c0db-9e79-4713-85dc-253dc0a72a6a kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Scaled up replica set success-200-isvc-6b9e1-predictor-77fcc484d9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:19Z" name: success-200-isvc-6b9e1-predictor.18a6ef69858d670c namespace: kserve-ci-e2e-test resourceVersion: "18719" uid: 2f083b96-55b6-46d9-9ab4-2181cb2ec62f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:12:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18712" uid: 115b97ff-7e2d-48b1-b2a0-0f7cd3747451 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-6b9e1-predictor.18a6ef6d03f100c9 namespace: kserve-ci-e2e-test resourceVersion: "19316" uid: 179632bf-bd2a-44d6-a092-959891e65dd1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:12:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18712" uid: 115b97ff-7e2d-48b1-b2a0-0f7cd3747451 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-6b9e1-predictor.18a6ef6d044ef014 namespace: kserve-ci-e2e-test resourceVersion: "19317" uid: a1c51152-03a6-4eee-b401-92263107b6af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: 115b97ff-7e2d-48b1-b2a0-0f7cd3747451 kind: Event lastTimestamp: "2026-04-16T20:13:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:34Z" name: success-200-isvc-6b9e1-predictor.18a6ef7780878f32 namespace: kserve-ci-e2e-test resourceVersion: "19634" uid: 33c6c9ac-6cb7-422c-899d-b23c4438543e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6b9e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: 115b97ff-7e2d-48b1-b2a0-0f7cd3747451 kind: Event lastTimestamp: "2026-04-16T20:13:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:13:34Z" name: success-200-isvc-6b9e1-predictor.18a6ef7780cae167 namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: f3e415b2-3c2b-4d1b-8e2e-58336211c594 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "18691" uid: 541945fc-75cc-409c-9a0a-9d4b66649755 kind: Event lastTimestamp: "2026-04-16T20:13:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:11Z" name: success-200-isvc-6b9e1.18a6ef698559ef8d namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: 82706655-b13b-4063-84e3-a4c2763e695d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6b9e1 namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: 541945fc-75cc-409c-9a0a-9d4b66649755 kind: Event lastTimestamp: "2026-04-16T20:13:11Z" message: InferenceService [success-200-isvc-6b9e1] is Ready metadata: creationTimestamp: "2026-04-16T20:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:13:11Z" name: success-200-isvc-6b9e1.18a6ef75af851251 namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 1cf69bab-7fc4-4282-80fc-ce838bf6013d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:02:54.619503Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b namespace: kserve-ci-e2e-test resourceVersion: "14376" uid: 000ed769-3a85-4a78-a087-5b689ec85d24 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:02:54Z" name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b.18a6eee5fdefc3d1 namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 725a60af-f50c-4f20-99b2-2ad62778ced7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b namespace: kserve-ci-e2e-test resourceVersion: "14394" uid: 000ed769-3a85-4a78-a087-5b689ec85d24 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:02:54Z" name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b.18a6eee60775b200 namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: 179f5ba9-b84d-4b61-a351-369fd9263848 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 000ed769-3a85-4a78-a087-5b689ec85d24 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b.18a6eee608ce8da7 namespace: kserve-ci-e2e-test resourceVersion: "14450" uid: 919e8e80-6519-4a89-826f-c73700a3fdfa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 000ed769-3a85-4a78-a087-5b689ec85d24 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b.18a6eee610d37a7a namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 65b8dc57-c290-43b4-88e6-080b7c96da90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 000ed769-3a85-4a78-a087-5b689ec85d24 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:54Z" name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b.18a6eee611497402 namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 9ab633ad-beec-43dd-bf0c-f06ae4140c9c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:02:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 000ed769-3a85-4a78-a087-5b689ec85d24 kind: Event lastTimestamp: "2026-04-16T20:03:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:02:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:03:36Z" name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b.18a6eee64502cab9 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 47ce2331-3272-43b8-9bf3-5ee3a6ab7bcf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 000ed769-3a85-4a78-a087-5b689ec85d24 kind: Event lastTimestamp: "2026-04-16T20:12:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:19Z" name: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b.18a6ef698786c49e namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: b92abf9c-f5ac-485f-b870-3737e67b02ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-99283-predictor-5b9fd86cdf namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: 4672aab0-6f1b-4984-bf89-716febff2091 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: 'Created pod: success-200-isvc-99283-predictor-5b9fd86cdf-j7k6b' metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:54Z" name: success-200-isvc-99283-predictor-5b9fd86cdf.18a6eee5fd62dde2 namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: 360e817b-d087-432b-98be-38f1871333ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: aeb0d8c7-2c93-4037-aea6-c499aa92e4b9 kind: Event lastTimestamp: "2026-04-16T20:02:54Z" message: Scaled up replica set success-200-isvc-99283-predictor-5b9fd86cdf from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:54Z" name: success-200-isvc-99283-predictor.18a6eee5fc0d0938 namespace: kserve-ci-e2e-test resourceVersion: "14366" uid: b39734f9-920d-4dfd-992a-8a19d1544aeb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:03:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 09a7ff30-2b76-4bd6-8431-a254c7c0fdc5 kind: Event lastTimestamp: "2026-04-16T20:03:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:24Z" name: success-200-isvc-99283-predictor.18a6eee97a83ac27 namespace: kserve-ci-e2e-test resourceVersion: "14731" uid: 248684e7-d8c1-4ae9-a64f-16d66becba21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:03:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 09a7ff30-2b76-4bd6-8431-a254c7c0fdc5 kind: Event lastTimestamp: "2026-04-16T20:03:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:24Z" name: success-200-isvc-99283-predictor.18a6eee97aeafc73 namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: c4cba12e-0188-4a2d-b1cb-c1803101bbc9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:03:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: 09a7ff30-2b76-4bd6-8431-a254c7c0fdc5 kind: Event lastTimestamp: "2026-04-16T20:03:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:54Z" name: success-200-isvc-99283-predictor.18a6eef0789b6b26 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 864b30d8-56d7-4816-b6c0-f7914df5a0f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:03:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99283-predictor namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: 09a7ff30-2b76-4bd6-8431-a254c7c0fdc5 kind: Event lastTimestamp: "2026-04-16T20:03:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:54Z" name: success-200-isvc-99283-predictor.18a6eef078f830b2 namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: 4be53f0a-4f6f-4b39-9ea8-c48f9dbab966 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:02:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-99283 namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: 220d14db-0f16-4f32-8154-d017acef7a30 kind: Event lastTimestamp: "2026-04-16T20:03:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:03:46Z" name: success-200-isvc-99283.18a6eee5fbfcae48 namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: 58e125a9-db9b-4714-915f-675b8dc47d31 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-99283 namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: 220d14db-0f16-4f32-8154-d017acef7a30 kind: Event lastTimestamp: "2026-04-16T20:03:46Z" message: InferenceService [success-200-isvc-99283] is Ready metadata: creationTimestamp: "2026-04-16T20:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:03:46Z" name: success-200-isvc-99283.18a6eef22ac02752 namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: 582ce8aa-13e1-4a46-8024-d4f6687a1ef8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:23:49.404508Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk namespace: kserve-ci-e2e-test resourceVersion: "24622" uid: 5bf97a5e-af1d-42e1-937a-53ff1be8f1a5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9a649-predictor-b857f97b5-fnrvk to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk.18a6f00a24f37472 namespace: kserve-ci-e2e-test resourceVersion: "24631" uid: ea7da936-cd41-4d03-bd3f-0dec559b8776 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk namespace: kserve-ci-e2e-test resourceVersion: "24639" uid: 5bf97a5e-af1d-42e1-937a-53ff1be8f1a5 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk.18a6f00a300383d8 namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: bb418ccc-44c9-4a0e-bd1c-ae97c3825153 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: 5bf97a5e-af1d-42e1-937a-53ff1be8f1a5 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk.18a6f00a3138fb52 namespace: kserve-ci-e2e-test resourceVersion: "24672" uid: cbb7a7e5-7d34-4983-b97c-92b8bc825baa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: 5bf97a5e-af1d-42e1-937a-53ff1be8f1a5 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk.18a6f00a392920fa namespace: kserve-ci-e2e-test resourceVersion: "24679" uid: e79e80d0-20ed-4ae4-a74d-fccc30acfbb6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: 5bf97a5e-af1d-42e1-937a-53ff1be8f1a5 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk.18a6f00a39a095a7 namespace: kserve-ci-e2e-test resourceVersion: "24680" uid: 3d6c950a-2229-4dd3-a3f2-b4b07d700187 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: 5bf97a5e-af1d-42e1-937a-53ff1be8f1a5 kind: Event lastTimestamp: "2026-04-16T20:24:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:24:31Z" name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk.18a6f00a4876c885 namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 5b4178ee-1913-49ac-9b9f-76659e62c4ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: 5bf97a5e-af1d-42e1-937a-53ff1be8f1a5 kind: Event lastTimestamp: "2026-04-16T20:33:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:33:14Z" name: success-200-isvc-9a649-predictor-b857f97b5-fnrvk.18a6f08dae550b44 namespace: kserve-ci-e2e-test resourceVersion: "29146" uid: ac270cdf-4057-4d34-81a8-2f8da767cda3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9a649-predictor-b857f97b5 namespace: kserve-ci-e2e-test resourceVersion: "24610" uid: d3e3ea73-ffcf-4479-8184-30960ff8836b kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: 'Created pod: success-200-isvc-9a649-predictor-b857f97b5-fnrvk' metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-9a649-predictor-b857f97b5.18a6f00a24711de7 namespace: kserve-ci-e2e-test resourceVersion: "24627" uid: 5d18f2ca-27a1-4fb3-8fb5-5a152c1587cc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24608" uid: 67ba851a-00c1-4319-9066-968142549a75 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Scaled up replica set success-200-isvc-9a649-predictor-b857f97b5 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-9a649-predictor.18a6f00a23218595 namespace: kserve-ci-e2e-test resourceVersion: "24612" uid: 20de98f8-f642-4565-b5d7-eb5103254ad8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24609" uid: f3ec8623-4317-4da7-a7b3-843e4da57241 kind: Event lastTimestamp: "2026-04-16T20:24:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:34Z" name: success-200-isvc-9a649-predictor.18a6f00da2167411 namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 573ad531-afac-4f43-8c60-5ae4de95c467 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24609" uid: f3ec8623-4317-4da7-a7b3-843e4da57241 kind: Event lastTimestamp: "2026-04-16T20:24:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:34Z" name: success-200-isvc-9a649-predictor.18a6f00da276df0b namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: 87348f23-029f-41c4-ae93-10706a90b1db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:24:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: f3ec8623-4317-4da7-a7b3-843e4da57241 kind: Event lastTimestamp: "2026-04-16T20:25:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:25:19Z" name: success-200-isvc-9a649-predictor.18a6f0181e8cf9b0 namespace: kserve-ci-e2e-test resourceVersion: "25512" uid: a63ec33d-9dc5-4263-b15b-2531e5cc0609 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:24:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a649-predictor namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: f3ec8623-4317-4da7-a7b3-843e4da57241 kind: Event lastTimestamp: "2026-04-16T20:25:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:25:19Z" name: success-200-isvc-9a649-predictor.18a6f0181effaa38 namespace: kserve-ci-e2e-test resourceVersion: "25513" uid: c62602ca-7c6c-4e70-8154-223fdf19c2a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9a649 namespace: kserve-ci-e2e-test resourceVersion: "24589" uid: 8bdb8b9d-f729-4c2a-ad40-f24b0b75c7ee kind: Event lastTimestamp: "2026-04-16T20:24:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:24:41Z" name: success-200-isvc-9a649.18a6f00a231bd4c3 namespace: kserve-ci-e2e-test resourceVersion: "25185" uid: fa8a50a7-8db7-42d9-b9b7-30c4c3558090 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9a649 namespace: kserve-ci-e2e-test resourceVersion: "25181" uid: 8bdb8b9d-f729-4c2a-ad40-f24b0b75c7ee kind: Event lastTimestamp: "2026-04-16T20:24:41Z" message: InferenceService [success-200-isvc-9a649] is Ready metadata: creationTimestamp: "2026-04-16T20:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:24:41Z" name: success-200-isvc-9a649.18a6f0162f1df0d7 namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: 8c95187c-0aa9-49fd-b96f-973f95e21d0e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:11:43.976408Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx namespace: kserve-ci-e2e-test resourceVersion: "18326" uid: 70a7fd93-0d3e-4360-bfdf-56ad797ba3c6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-de58a-predictor-5959f57c5c-pfhmx to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:11:43Z" name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx.18a6ef613e1124bd namespace: kserve-ci-e2e-test resourceVersion: "18342" uid: 08200bd7-0e96-4164-8868-91c525f59cd9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx namespace: kserve-ci-e2e-test resourceVersion: "18355" uid: 70a7fd93-0d3e-4360-bfdf-56ad797ba3c6 kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:11:44Z" name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx.18a6ef6147c3f7bd namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: d19307e7-7e3f-4388-b912-593b6db8c1d8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx namespace: kserve-ci-e2e-test resourceVersion: "18333" uid: 70a7fd93-0d3e-4360-bfdf-56ad797ba3c6 kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:44Z" name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx.18a6ef61496b6540 namespace: kserve-ci-e2e-test resourceVersion: "18371" uid: 038514f8-87d4-4cd8-844e-7da7134085f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx namespace: kserve-ci-e2e-test resourceVersion: "18333" uid: 70a7fd93-0d3e-4360-bfdf-56ad797ba3c6 kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:44Z" name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx.18a6ef614d80a362 namespace: kserve-ci-e2e-test resourceVersion: "18378" uid: 8ba0361d-a40f-492e-948b-bebe8c34e813 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx namespace: kserve-ci-e2e-test resourceVersion: "18333" uid: 70a7fd93-0d3e-4360-bfdf-56ad797ba3c6 kind: Event lastTimestamp: "2026-04-16T20:11:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:44Z" name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx.18a6ef614e0b1cd7 namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: f998862e-8861-40cb-b141-10e5817a9d77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx namespace: kserve-ci-e2e-test resourceVersion: "18333" uid: 70a7fd93-0d3e-4360-bfdf-56ad797ba3c6 kind: Event lastTimestamp: "2026-04-16T20:13:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:05Z" name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx.18a6ef6162225335 namespace: kserve-ci-e2e-test resourceVersion: "19342" uid: 7524341a-a17f-4255-b31a-50046f5f26c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx namespace: kserve-ci-e2e-test resourceVersion: "18333" uid: 70a7fd93-0d3e-4360-bfdf-56ad797ba3c6 kind: Event lastTimestamp: "2026-04-16T20:13:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:13:04Z" name: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx.18a6ef73eddd0697 namespace: kserve-ci-e2e-test resourceVersion: "19278" uid: 8d734df1-207d-492f-92e8-fc0bdb1937f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-de58a-predictor-5959f57c5c namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: f8d9c835-2815-41fe-bc1b-6866891f4f4f kind: Event lastTimestamp: "2026-04-16T20:11:43Z" message: 'Created pod: success-200-isvc-de58a-predictor-5959f57c5c-pfhmx' metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:43Z" name: success-200-isvc-de58a-predictor-5959f57c5c.18a6ef613d70452b namespace: kserve-ci-e2e-test resourceVersion: "18331" uid: 22900a5e-9cfa-4878-84c6-33fb56bef0eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18303" uid: 4eaa6224-1121-4bac-9ec7-5054495aa2b1 kind: Event lastTimestamp: "2026-04-16T20:11:43Z" message: Scaled up replica set success-200-isvc-de58a-predictor-5959f57c5c from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:11:43Z" name: success-200-isvc-de58a-predictor.18a6ef6139ea04db namespace: kserve-ci-e2e-test resourceVersion: "18311" uid: 372b72c2-6377-4fde-8af4-7d2f6de6d714 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:11:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18308" uid: fd5eb8d8-dad5-4254-ac1d-ef3837a0cf0e kind: Event lastTimestamp: "2026-04-16T20:12:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:13Z" name: success-200-isvc-de58a-predictor.18a6ef64b83f901f namespace: kserve-ci-e2e-test resourceVersion: "18623" uid: 1246f00d-b99e-49ff-bad0-18f7d0e8bd5b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:11:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18308" uid: fd5eb8d8-dad5-4254-ac1d-ef3837a0cf0e kind: Event lastTimestamp: "2026-04-16T20:12:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:13Z" name: success-200-isvc-de58a-predictor.18a6ef64b8a423bd namespace: kserve-ci-e2e-test resourceVersion: "18624" uid: a39c003b-19fd-4230-bc67-bbe89aff8a16 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:12:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18514" uid: fd5eb8d8-dad5-4254-ac1d-ef3837a0cf0e kind: Event lastTimestamp: "2026-04-16T20:12:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:12:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:58Z" name: success-200-isvc-de58a-predictor.18a6ef6bb6809dc3 namespace: kserve-ci-e2e-test resourceVersion: "19158" uid: 179de37d-759f-4265-9519-debf523f36ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T20:12:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de58a-predictor namespace: kserve-ci-e2e-test resourceVersion: "18514" uid: fd5eb8d8-dad5-4254-ac1d-ef3837a0cf0e kind: Event lastTimestamp: "2026-04-16T20:12:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:12:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:12:58Z" name: success-200-isvc-de58a-predictor.18a6ef6bb6e57378 namespace: kserve-ci-e2e-test resourceVersion: "19159" uid: ec5b3744-ef87-4e1e-ad8b-f04bd3c78226 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de58a namespace: kserve-ci-e2e-test resourceVersion: "18289" uid: d327e968-af28-4e23-8b74-cba78f1320e3 kind: Event lastTimestamp: "2026-04-16T20:12:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:12:35Z" name: success-200-isvc-de58a.18a6ef6139a951aa namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: 958effa7-b184-4909-a251-9cebb658b548 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de58a namespace: kserve-ci-e2e-test resourceVersion: "18289" uid: d327e968-af28-4e23-8b74-cba78f1320e3 kind: Event lastTimestamp: "2026-04-16T20:11:43Z" message: 'Failed to update status for InferenceService "success-200-isvc-de58a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-de58a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:11:43Z" name: success-200-isvc-de58a.18a6ef613a179114 namespace: kserve-ci-e2e-test resourceVersion: "18314" uid: a3f2abdb-1198-4751-9eec-04aacf3dd36e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de58a namespace: kserve-ci-e2e-test resourceVersion: "18289" uid: d327e968-af28-4e23-8b74-cba78f1320e3 kind: Event lastTimestamp: "2026-04-16T20:11:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-de58a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:11:43Z" name: success-200-isvc-de58a.18a6ef613a17e5ad namespace: kserve-ci-e2e-test resourceVersion: "18315" uid: 53de4796-7c1b-4e9e-ac15-d239b3fdb5c7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:12:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de58a namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: d327e968-af28-4e23-8b74-cba78f1320e3 kind: Event lastTimestamp: "2026-04-16T20:12:35Z" message: InferenceService [success-200-isvc-de58a] is Ready metadata: creationTimestamp: "2026-04-16T20:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:12:35Z" name: success-200-isvc-de58a.18a6ef6d466a7e31 namespace: kserve-ci-e2e-test resourceVersion: "18939" uid: 12d4f34d-b2d5-4f75-8e62-f4bf38aa1c02 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:22:29.121269Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb namespace: kserve-ci-e2e-test resourceVersion: "23691" uid: cbee9c44-d672-4fc6-81bf-67c5c6b666c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb.18a6eff773b2856a namespace: kserve-ci-e2e-test resourceVersion: "23707" uid: 4923d684-7d5a-447d-9ef6-18dd6045afd3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb namespace: kserve-ci-e2e-test resourceVersion: "23721" uid: cbee9c44-d672-4fc6-81bf-67c5c6b666c9 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb.18a6eff77d0c44e9 namespace: kserve-ci-e2e-test resourceVersion: "23734" uid: e0f74f8e-7fbb-4cd5-9dfd-aded440fde06 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb namespace: kserve-ci-e2e-test resourceVersion: "23697" uid: cbee9c44-d672-4fc6-81bf-67c5c6b666c9 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb.18a6eff77e668f0b namespace: kserve-ci-e2e-test resourceVersion: "23737" uid: e406dda2-bf5d-46a2-b778-d283ceb8009e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb namespace: kserve-ci-e2e-test resourceVersion: "23697" uid: cbee9c44-d672-4fc6-81bf-67c5c6b666c9 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb.18a6eff787403bc6 namespace: kserve-ci-e2e-test resourceVersion: "23748" uid: 970f0848-2659-4835-8fdc-e499d00dcea3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb namespace: kserve-ci-e2e-test resourceVersion: "23697" uid: cbee9c44-d672-4fc6-81bf-67c5c6b666c9 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb.18a6eff787b6cddf namespace: kserve-ci-e2e-test resourceVersion: "23749" uid: 6b9a9d42-b76f-4c0e-a72c-dd36f3198b5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb namespace: kserve-ci-e2e-test resourceVersion: "23697" uid: cbee9c44-d672-4fc6-81bf-67c5c6b666c9 kind: Event lastTimestamp: "2026-04-16T20:23:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:50Z" name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb.18a6eff796bb3833 namespace: kserve-ci-e2e-test resourceVersion: "24707" uid: b33f51b5-84ec-4626-a63e-570569cdbc22 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb namespace: kserve-ci-e2e-test resourceVersion: "23697" uid: cbee9c44-d672-4fc6-81bf-67c5c6b666c9 kind: Event lastTimestamp: "2026-04-16T20:23:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:23:49Z" name: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb.18a6f00a25d52681 namespace: kserve-ci-e2e-test resourceVersion: "24637" uid: d74a6205-6b6e-43a6-a8cc-9c2d6dc5980d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e8b6e-predictor-66bb45779f namespace: kserve-ci-e2e-test resourceVersion: "23672" uid: 28f79a58-6819-4a69-8be4-e442f9e6fe01 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: 'Created pod: success-200-isvc-e8b6e-predictor-66bb45779f-wmmzb' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-e8b6e-predictor-66bb45779f.18a6eff7732f27da namespace: kserve-ci-e2e-test resourceVersion: "23696" uid: 825d10aa-7953-4260-bafa-c832b5161e5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23665" uid: 3c456776-e7ae-48c4-a156-05739f8378c4 kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: Scaled up replica set success-200-isvc-e8b6e-predictor-66bb45779f from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-e8b6e-predictor.18a6eff76fbf8209 namespace: kserve-ci-e2e-test resourceVersion: "23676" uid: 52cf6880-b68c-4462-9095-35dea615e50d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23671" uid: 9b89c795-eb02-4451-ba40-17346684dfb0 kind: Event lastTimestamp: "2026-04-16T20:22:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:22:44Z" name: success-200-isvc-e8b6e-predictor.18a6effaee78ab6f namespace: kserve-ci-e2e-test resourceVersion: "23870" uid: ddd54a58-c3a5-4242-b7ae-c368e5eb3aee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23671" uid: 9b89c795-eb02-4451-ba40-17346684dfb0 kind: Event lastTimestamp: "2026-04-16T20:22:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:22:44Z" name: success-200-isvc-e8b6e-predictor.18a6effaeee280c1 namespace: kserve-ci-e2e-test resourceVersion: "23872" uid: 3965c351-fd87-40cd-a5e9-8289a393e6e8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23871" uid: 9b89c795-eb02-4451-ba40-17346684dfb0 kind: Event lastTimestamp: "2026-04-16T20:23:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:44Z" name: success-200-isvc-e8b6e-predictor.18a6effe6d952b1d namespace: kserve-ci-e2e-test resourceVersion: "24521" uid: 17d96d79-e1e7-4a5f-8bc1-a838990b4e4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "23871" uid: 9b89c795-eb02-4451-ba40-17346684dfb0 kind: Event lastTimestamp: "2026-04-16T20:23:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:23:44Z" name: success-200-isvc-e8b6e-predictor.18a6effe6e01c340 namespace: kserve-ci-e2e-test resourceVersion: "24522" uid: 8c510b4a-0871-478e-b53d-370e6effcc18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "23652" uid: 8cb6adcc-a7c7-4d2f-8a7e-f1602cee17bb kind: Event lastTimestamp: "2026-04-16T20:23:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:20Z" name: success-200-isvc-e8b6e.18a6eff76f9f666e namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: 40a5aa72-c2e7-4196-be72-4a1697e3dafa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "23652" uid: 8cb6adcc-a7c7-4d2f-8a7e-f1602cee17bb kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-e8b6e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e8b6e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-e8b6e.18a6eff77005c400 namespace: kserve-ci-e2e-test resourceVersion: "23678" uid: d867b4f8-5489-47f5-9245-a163da4cb972 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:22:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "23652" uid: 8cb6adcc-a7c7-4d2f-8a7e-f1602cee17bb kind: Event lastTimestamp: "2026-04-16T20:22:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e8b6e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T20:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:22:29Z" name: success-200-isvc-e8b6e.18a6eff770062787 namespace: kserve-ci-e2e-test resourceVersion: "23679" uid: f8c08c74-6c00-4823-b51e-511c1b7df699 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:23:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8b6e namespace: kserve-ci-e2e-test resourceVersion: "24305" uid: 8cb6adcc-a7c7-4d2f-8a7e-f1602cee17bb kind: Event lastTimestamp: "2026-04-16T20:23:20Z" message: InferenceService [success-200-isvc-e8b6e] is Ready metadata: creationTimestamp: "2026-04-16T20:23:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:23:20Z" name: success-200-isvc-e8b6e.18a6f0037c9832aa namespace: kserve-ci-e2e-test resourceVersion: "24306" uid: 1afab8cc-7390-4d12-8a09-89e2ed097c86 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:02:04.364281Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1bf21-b5cd598bc-pm5nx namespace: kserve-ci-e2e-test resourceVersion: "13692" uid: c738934a-b6dc-4ce0-9172-802752c19936 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1bf21-b5cd598bc-pm5nx to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:02:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:02:04Z" name: switch-graph-1bf21-b5cd598bc-pm5nx.18a6eeda4a7de902 namespace: kserve-ci-e2e-test resourceVersion: "13698" uid: 8309d644-abfc-4a69-867b-dca286e6a34f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:04Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1bf21-b5cd598bc-pm5nx namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: c738934a-b6dc-4ce0-9172-802752c19936 kind: Event lastTimestamp: "2026-04-16T20:02:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-1bf21-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:02:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:04Z" name: switch-graph-1bf21-b5cd598bc-pm5nx.18a6eeda54e0ffb6 namespace: kserve-ci-e2e-test resourceVersion: "13709" uid: 59af5bbb-b068-44a2-9317-bdba74802119 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:05Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1bf21-b5cd598bc-pm5nx namespace: kserve-ci-e2e-test resourceVersion: "13705" uid: c738934a-b6dc-4ce0-9172-802752c19936 kind: Event lastTimestamp: "2026-04-16T20:02:05Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:02:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:02:05Z" name: switch-graph-1bf21-b5cd598bc-pm5nx.18a6eeda88d17c86 namespace: kserve-ci-e2e-test resourceVersion: "13718" uid: e59190f1-1309-47e1-8d60-747c3eda454b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1bf21} kind: Pod name: switch-graph-1bf21-b5cd598bc-pm5nx namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: c738934a-b6dc-4ce0-9172-802752c19936 kind: Event lastTimestamp: "2026-04-16T20:02:05Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" metadata: creationTimestamp: "2026-04-16T20:02:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:05Z" name: switch-graph-1bf21-b5cd598bc-pm5nx.18a6eeda8a037a99 namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: 2c97d440-331c-4225-9cb4-8556ae1d3c87 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1bf21} kind: Pod name: switch-graph-1bf21-b5cd598bc-pm5nx namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: c738934a-b6dc-4ce0-9172-802752c19936 kind: Event lastTimestamp: "2026-04-16T20:02:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" in 2.525s (2.525s including waiting). Image size: 216232023 bytes.' metadata: creationTimestamp: "2026-04-16T20:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:07Z" name: switch-graph-1bf21-b5cd598bc-pm5nx.18a6eedb20907a47 namespace: kserve-ci-e2e-test resourceVersion: "13739" uid: ba434ef3-6d88-4b13-9a49-d212ebba2c37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1bf21} kind: Pod name: switch-graph-1bf21-b5cd598bc-pm5nx namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: c738934a-b6dc-4ce0-9172-802752c19936 kind: Event lastTimestamp: "2026-04-16T20:02:08Z" message: 'Created container: switch-graph-1bf21' metadata: creationTimestamp: "2026-04-16T20:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:08Z" name: switch-graph-1bf21-b5cd598bc-pm5nx.18a6eedb2386e850 namespace: kserve-ci-e2e-test resourceVersion: "13740" uid: b5781105-1535-4f0e-9b16-42b23283f65f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1bf21} kind: Pod name: switch-graph-1bf21-b5cd598bc-pm5nx namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: c738934a-b6dc-4ce0-9172-802752c19936 kind: Event lastTimestamp: "2026-04-16T20:02:08Z" message: Started container switch-graph-1bf21 metadata: creationTimestamp: "2026-04-16T20:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:08Z" name: switch-graph-1bf21-b5cd598bc-pm5nx.18a6eedb23f6a7ac namespace: kserve-ci-e2e-test resourceVersion: "13741" uid: 27b3c15a-fa2e-4cd3-ab28-d203e63ada23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1bf21} kind: Pod name: switch-graph-1bf21-b5cd598bc-pm5nx namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: c738934a-b6dc-4ce0-9172-802752c19936 kind: Event lastTimestamp: "2026-04-16T20:02:18Z" message: Stopping container switch-graph-1bf21 metadata: creationTimestamp: "2026-04-16T20:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:18Z" name: switch-graph-1bf21-b5cd598bc-pm5nx.18a6eedd9f52945f namespace: kserve-ci-e2e-test resourceVersion: "13862" uid: 35019c4a-2ee1-4476-9af8-301564c51d90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1bf21} kind: Pod name: switch-graph-1bf21-b5cd598bc-pm5nx namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: c738934a-b6dc-4ce0-9172-802752c19936 kind: Event lastTimestamp: "2026-04-16T20:02:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:02:44Z" name: switch-graph-1bf21-b5cd598bc-pm5nx.18a6eeddd52f324f namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: 4ca844f9-0510-4853-a34f-e6e103d62f7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1bf21-b5cd598bc namespace: kserve-ci-e2e-test resourceVersion: "13689" uid: 71466cbd-e080-44a4-9ace-1b2da8585c63 kind: Event lastTimestamp: "2026-04-16T20:02:04Z" message: 'Created pod: switch-graph-1bf21-b5cd598bc-pm5nx' metadata: creationTimestamp: "2026-04-16T20:02:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:04Z" name: switch-graph-1bf21-b5cd598bc.18a6eeda49f13956 namespace: kserve-ci-e2e-test resourceVersion: "13693" uid: 098cb835-3c1c-4a36-bdaa-88dff66f55dd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1bf21 namespace: kserve-ci-e2e-test resourceVersion: "13687" uid: 58223473-4764-4fe2-b1a4-28ea37ee8ede kind: Event lastTimestamp: "2026-04-16T20:02:04Z" message: Scaled up replica set switch-graph-1bf21-b5cd598bc from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:02:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:02:04Z" name: switch-graph-1bf21.18a6eeda4932a274 namespace: kserve-ci-e2e-test resourceVersion: "13690" uid: 27cb229a-a439-4b85-ad26-86d36ce5864b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:02:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1bf21 namespace: kserve-ci-e2e-test resourceVersion: "13702" uid: fa4f8f39-3a07-4ab8-a6b2-464e1ca8f8e8 kind: Event lastTimestamp: "2026-04-16T20:02:04Z" message: InferenceGraph [switch-graph-1bf21] is Ready metadata: creationTimestamp: "2026-04-16T20:02:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:02:04Z" name: switch-graph-1bf21.18a6eeda4c8198db namespace: kserve-ci-e2e-test resourceVersion: "13704" uid: 5b3c6769-ff92-4dee-b389-e3a513beb977 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:03:28.913853Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3d0d2-5f445cd894-whsfp namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: d15cb7d3-244f-4259-9938-976efcabc968 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3d0d2-5f445cd894-whsfp to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:03:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:03:28Z" name: switch-graph-3d0d2-5f445cd894-whsfp.18a6eeedfa09ff70 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 5787f10e-5f26-4730-b0ad-78b05991546e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:29Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3d0d2-5f445cd894-whsfp namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: d15cb7d3-244f-4259-9938-976efcabc968 kind: Event lastTimestamp: "2026-04-16T20:03:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-3d0d2-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:03:29Z" name: switch-graph-3d0d2-5f445cd894-whsfp.18a6eeee048220e2 namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: 89c449b7-3f5b-47d6-b02e-7be173ff0d72 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:29Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3d0d2-5f445cd894-whsfp namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: d15cb7d3-244f-4259-9938-976efcabc968 kind: Event lastTimestamp: "2026-04-16T20:03:29Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:03:29Z" name: switch-graph-3d0d2-5f445cd894-whsfp.18a6eeee36ca93e1 namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: d323d094-c0c1-43d4-8c34-b29eaa296905 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d0d2} kind: Pod name: switch-graph-3d0d2-5f445cd894-whsfp namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: d15cb7d3-244f-4259-9938-976efcabc968 kind: Event lastTimestamp: "2026-04-16T20:03:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:03:29Z" name: switch-graph-3d0d2-5f445cd894-whsfp.18a6eeee37f259ce namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: c0abd436-6b34-445d-9b4d-9919ce500705 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d0d2} kind: Pod name: switch-graph-3d0d2-5f445cd894-whsfp namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: d15cb7d3-244f-4259-9938-976efcabc968 kind: Event lastTimestamp: "2026-04-16T20:03:30Z" message: 'Created container: switch-graph-3d0d2' metadata: creationTimestamp: "2026-04-16T20:03:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:03:30Z" name: switch-graph-3d0d2-5f445cd894-whsfp.18a6eeee3acb5523 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: 968e7782-752d-4f4d-8ca2-fdf0c120857c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d0d2} kind: Pod name: switch-graph-3d0d2-5f445cd894-whsfp namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: d15cb7d3-244f-4259-9938-976efcabc968 kind: Event lastTimestamp: "2026-04-16T20:03:30Z" message: Started container switch-graph-3d0d2 metadata: creationTimestamp: "2026-04-16T20:03:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:03:30Z" name: switch-graph-3d0d2-5f445cd894-whsfp.18a6eeee3b369d52 namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: fb4a035b-975c-4421-b83a-31669cfabc9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d0d2} kind: Pod name: switch-graph-3d0d2-5f445cd894-whsfp namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: d15cb7d3-244f-4259-9938-976efcabc968 kind: Event lastTimestamp: "2026-04-16T20:11:43Z" message: Stopping container switch-graph-3d0d2 metadata: creationTimestamp: "2026-04-16T20:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:11:43Z" name: switch-graph-3d0d2-5f445cd894-whsfp.18a6ef6130249101 namespace: kserve-ci-e2e-test resourceVersion: "18285" uid: d02f5563-7698-424c-8b5a-683756cf5379 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T20:11:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3d0d2} kind: Pod name: switch-graph-3d0d2-5f445cd894-whsfp namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: d15cb7d3-244f-4259-9938-976efcabc968 kind: Event lastTimestamp: "2026-04-16T20:12:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:12:11Z" name: switch-graph-3d0d2-5f445cd894-whsfp.18a6ef61e91e0ea9 namespace: kserve-ci-e2e-test resourceVersion: "18605" uid: 80bbe423-d351-4c25-8796-cb1368f29c83 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3d0d2-5f445cd894 namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 13ebcbaa-5a51-4de5-9e5a-11cc1e0431ed kind: Event lastTimestamp: "2026-04-16T20:03:28Z" message: 'Created pod: switch-graph-3d0d2-5f445cd894-whsfp' metadata: creationTimestamp: "2026-04-16T20:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:28Z" name: switch-graph-3d0d2-5f445cd894.18a6eeedf9a0340b namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 277fb33c-0520-4491-9973-87031340882c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: d39dec62-eb89-41ba-9916-535839c2109d kind: Event lastTimestamp: "2026-04-16T20:03:28Z" message: Scaled up replica set switch-graph-3d0d2-5f445cd894 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:28Z" name: switch-graph-3d0d2.18a6eeedf8b89510 namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: 52bb1440-4dee-450c-a0c3-ee3427477bf8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 6a9d65bc-fae4-4c2e-8d36-bc7580195795 kind: Event lastTimestamp: "2026-04-16T20:03:28Z" message: InferenceGraph [switch-graph-3d0d2] is Ready metadata: creationTimestamp: "2026-04-16T20:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:03:28Z" name: switch-graph-3d0d2.18a6eeedfcf2253e namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: 59c60b1d-35ed-408b-9e3b-040ef25961e7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: f8940a2f-4e33-4efe-b63e-3518f88b49f3 kind: Event lastTimestamp: "2026-04-16T20:03:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:03:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:43Z" name: switch-graph-3d0d2.18a6eef177291ba7 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 6be8f1e1-2331-4901-9e62-6402862f981c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: f8940a2f-4e33-4efe-b63e-3518f88b49f3 kind: Event lastTimestamp: "2026-04-16T20:03:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:03:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:43Z" name: switch-graph-3d0d2.18a6eef1778eede9 namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: a7c02edb-04fb-424f-a335-0a18b4b41e33 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: f8940a2f-4e33-4efe-b63e-3518f88b49f3 kind: Event lastTimestamp: "2026-04-16T20:03:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:58Z" name: switch-graph-3d0d2.18a6eef4f64107f6 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 5865bb97-fc0a-4701-9bf1-cc4fb57f00f0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:03:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3d0d2 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: f8940a2f-4e33-4efe-b63e-3518f88b49f3 kind: Event lastTimestamp: "2026-04-16T20:03:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T20:03:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:03:58Z" name: switch-graph-3d0d2.18a6eef4f6aa99b3 namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 80560690-d2ab-4680-ace0-b2ab2d70f7fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T20:24:14.838176Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-653df-fd48d9cd8-p5dbm namespace: kserve-ci-e2e-test resourceVersion: "24931" uid: 53913be0-8d2a-4426-87ed-b208eb235786 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-653df-fd48d9cd8-p5dbm to ip-10-0-140-191.ec2.internal metadata: creationTimestamp: "2026-04-16T20:24:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T20:24:14Z" name: switch-graph-653df-fd48d9cd8-p5dbm.18a6f01010ea756b namespace: kserve-ci-e2e-test resourceVersion: "24936" uid: 3e83e02f-56e9-4325-a860-4942dc95ebb5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d469f96fb-nblbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:15Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-653df-fd48d9cd8-p5dbm namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 53913be0-8d2a-4426-87ed-b208eb235786 kind: Event lastTimestamp: "2026-04-16T20:24:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-653df-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:24:15Z" name: switch-graph-653df-fd48d9cd8-p5dbm.18a6f0101f254d38 namespace: kserve-ci-e2e-test resourceVersion: "24948" uid: b733ab90-2ff3-4501-8ecd-f3a29248ea7d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:15Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-653df-fd48d9cd8-p5dbm namespace: kserve-ci-e2e-test resourceVersion: "24940" uid: 53913be0-8d2a-4426-87ed-b208eb235786 kind: Event lastTimestamp: "2026-04-16T20:24:15Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T20:24:15Z" name: switch-graph-653df-fd48d9cd8-p5dbm.18a6f0104d8c7574 namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 320e9780-186b-46ba-b071-02dfa46cb4de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-653df} kind: Pod name: switch-graph-653df-fd48d9cd8-p5dbm namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 53913be0-8d2a-4426-87ed-b208eb235786 kind: Event lastTimestamp: "2026-04-16T20:24:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:24:15Z" name: switch-graph-653df-fd48d9cd8-p5dbm.18a6f0104ec7907d namespace: kserve-ci-e2e-test resourceVersion: "24957" uid: 2a67c3f3-c728-41f4-a855-b37e30d245b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-653df} kind: Pod name: switch-graph-653df-fd48d9cd8-p5dbm namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 53913be0-8d2a-4426-87ed-b208eb235786 kind: Event lastTimestamp: "2026-04-16T20:24:15Z" message: 'Created container: switch-graph-653df' metadata: creationTimestamp: "2026-04-16T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:24:15Z" name: switch-graph-653df-fd48d9cd8-p5dbm.18a6f010518aa0c5 namespace: kserve-ci-e2e-test resourceVersion: "24958" uid: c85bb1f7-2f03-4b13-b759-3636b0dbb906 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-653df} kind: Pod name: switch-graph-653df-fd48d9cd8-p5dbm namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 53913be0-8d2a-4426-87ed-b208eb235786 kind: Event lastTimestamp: "2026-04-16T20:24:15Z" message: Started container switch-graph-653df metadata: creationTimestamp: "2026-04-16T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:24:15Z" name: switch-graph-653df-fd48d9cd8-p5dbm.18a6f01051f9b043 namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 52b616ba-967d-4d1e-8b1f-b191e9e2a988 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-653df} kind: Pod name: switch-graph-653df-fd48d9cd8-p5dbm namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 53913be0-8d2a-4426-87ed-b208eb235786 kind: Event lastTimestamp: "2026-04-16T20:40:34Z" message: Stopping container switch-graph-653df metadata: creationTimestamp: "2026-04-16T20:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:40:34Z" name: switch-graph-653df-fd48d9cd8-p5dbm.18a6f0f41b052e48 namespace: kserve-ci-e2e-test resourceVersion: "32184" uid: 8b6f21aa-3296-41e9-b9de-5107edf08b42 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T20:40:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-653df} kind: Pod name: switch-graph-653df-fd48d9cd8-p5dbm namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 53913be0-8d2a-4426-87ed-b208eb235786 kind: Event lastTimestamp: "2026-04-16T20:40:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T20:40:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T20:40:52Z" name: switch-graph-653df-fd48d9cd8-p5dbm.18a6f0f4db3c052d namespace: kserve-ci-e2e-test resourceVersion: "32386" uid: 3cfc01bd-0cca-4f98-80f3-5fe3ebc2d9d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-191.ec2.internal source: component: kubelet host: ip-10-0-140-191.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-653df-fd48d9cd8 namespace: kserve-ci-e2e-test resourceVersion: "24928" uid: 6274d66d-f834-4389-b2cd-f94b64c88cf4 kind: Event lastTimestamp: "2026-04-16T20:24:14Z" message: 'Created pod: switch-graph-653df-fd48d9cd8-p5dbm' metadata: creationTimestamp: "2026-04-16T20:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:14Z" name: switch-graph-653df-fd48d9cd8.18a6f01010527606 namespace: kserve-ci-e2e-test resourceVersion: "24933" uid: 17533d14-e278-47d4-853b-ab58fbddb113 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-653df namespace: kserve-ci-e2e-test resourceVersion: "24926" uid: 8c8542f7-b010-446c-95ff-56b0e46fda72 kind: Event lastTimestamp: "2026-04-16T20:24:14Z" message: Scaled up replica set switch-graph-653df-fd48d9cd8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T20:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:14Z" name: switch-graph-653df.18a6f0100f8d03f6 namespace: kserve-ci-e2e-test resourceVersion: "24929" uid: c8a3a5f3-9a6d-403f-8486-4b8489ec1227 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T20:24:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-653df namespace: kserve-ci-e2e-test resourceVersion: "24942" uid: c9c482e5-9d05-49c1-89ac-430fa3a61839 kind: Event lastTimestamp: "2026-04-16T20:24:14Z" message: InferenceGraph [switch-graph-653df] is Ready metadata: creationTimestamp: "2026-04-16T20:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T20:24:14Z" name: switch-graph-653df.18a6f010131749a1 namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: d51285ef-5d34-4a96-84aa-f7b650bd9492 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:24:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-653df namespace: kserve-ci-e2e-test resourceVersion: "24927" uid: 8c5b6d69-e3de-4fbe-83c4-8b03c52ce7fa kind: Event lastTimestamp: "2026-04-16T20:24:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:44Z" name: switch-graph-653df.18a6f0138e41368a namespace: kserve-ci-e2e-test resourceVersion: "25211" uid: b8d4ee25-7de1-4e8e-8d10-53b7abcf3657 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T20:24:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-653df namespace: kserve-ci-e2e-test resourceVersion: "24927" uid: 8c5b6d69-e3de-4fbe-83c4-8b03c52ce7fa kind: Event lastTimestamp: "2026-04-16T20:24:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T20:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T20:24:44Z" name: switch-graph-653df.18a6f0138e9a4e44 namespace: kserve-ci-e2e-test resourceVersion: "25212" uid: 36f53193-1b8f-4cb1-b5b4-11fa7e90eb58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32413"