--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:31:28.140690Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bac14-959f8bf68-g78lw namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 9f90f449-829f-41cb-822c-83380f6743bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-bac14-959f8bf68-g78lw to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:31:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:31:28Z" name: ensemble-graph-bac14-959f8bf68-g78lw.18a6e9e89250d49c namespace: kserve-ci-e2e-test resourceVersion: "20707" uid: 36bb9736-ebd9-488b-8e88-13fb772c16ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:28Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bac14-959f8bf68-g78lw namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 9f90f449-829f-41cb-822c-83380f6743bc kind: Event lastTimestamp: "2026-04-16T18:31:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-bac14-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:28Z" name: ensemble-graph-bac14-959f8bf68-g78lw.18a6e9e89aaff7a0 namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 6d6aabb2-9f47-4ef9-abf1-c761d0128d5b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:29Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bac14-959f8bf68-g78lw namespace: kserve-ci-e2e-test resourceVersion: "20711" uid: 9f90f449-829f-41cb-822c-83380f6743bc kind: Event lastTimestamp: "2026-04-16T18:31:29Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:31:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:31:29Z" name: ensemble-graph-bac14-959f8bf68-g78lw.18a6e9e8cef47191 namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: 5bf42fef-c10f-4808-a122-2eb43c5c09fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bac14} kind: Pod name: ensemble-graph-bac14-959f8bf68-g78lw namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 9f90f449-829f-41cb-822c-83380f6743bc kind: Event lastTimestamp: "2026-04-16T18:31:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:31:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:29Z" name: ensemble-graph-bac14-959f8bf68-g78lw.18a6e9e8d071caf5 namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: 3ea5d3e1-bf7e-44f8-9392-1d95146b55e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bac14} kind: Pod name: ensemble-graph-bac14-959f8bf68-g78lw namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 9f90f449-829f-41cb-822c-83380f6743bc kind: Event lastTimestamp: "2026-04-16T18:31:29Z" message: 'Created container: ensemble-graph-bac14' metadata: creationTimestamp: "2026-04-16T18:31:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:29Z" name: ensemble-graph-bac14-959f8bf68-g78lw.18a6e9e8d3b0f34b namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: e14b6b82-43f7-4496-95d9-ffec568115bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bac14} kind: Pod name: ensemble-graph-bac14-959f8bf68-g78lw namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 9f90f449-829f-41cb-822c-83380f6743bc kind: Event lastTimestamp: "2026-04-16T18:31:29Z" message: Started container ensemble-graph-bac14 metadata: creationTimestamp: "2026-04-16T18:31:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:29Z" name: ensemble-graph-bac14-959f8bf68-g78lw.18a6e9e8d42a6f0f namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: f525ba8c-d4d3-4dd9-a919-16a0d2932668 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bac14} kind: Pod name: ensemble-graph-bac14-959f8bf68-g78lw namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 9f90f449-829f-41cb-822c-83380f6743bc kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: Stopping container ensemble-graph-bac14 metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:42Z" name: ensemble-graph-bac14-959f8bf68-g78lw.18a6ea5bb914888c namespace: kserve-ci-e2e-test resourceVersion: "24211" uid: 270d29ed-3e01-4845-9c2d-183cd917da44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bac14} kind: Pod name: ensemble-graph-bac14-959f8bf68-g78lw namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 9f90f449-829f-41cb-822c-83380f6743bc kind: Event lastTimestamp: "2026-04-16T18:40:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:10Z" name: ensemble-graph-bac14-959f8bf68-g78lw.18a6ea5c7957c56e namespace: kserve-ci-e2e-test resourceVersion: "24536" uid: 0f3b415f-c930-46fc-891e-baaa0c131a7b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-bac14-959f8bf68 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 948d7c38-988d-4aee-ab6d-8d2bf56205dd kind: Event lastTimestamp: "2026-04-16T18:31:28Z" message: 'Created pod: ensemble-graph-bac14-959f8bf68-g78lw' metadata: creationTimestamp: "2026-04-16T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:28Z" name: ensemble-graph-bac14-959f8bf68.18a6e9e891bf1a52 namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: 4f2fa62b-d05a-485d-a5f4-8cc1b6de97a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-bac14 namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 6c8dbea3-bd83-4fbd-9be7-b73a974719eb kind: Event lastTimestamp: "2026-04-16T18:31:28Z" message: Scaled up replica set ensemble-graph-bac14-959f8bf68 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:28Z" name: ensemble-graph-bac14.18a6e9e890b6bae2 namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: af2aeae9-04af-4c27-a464-082d17f22a56 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-bac14 namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: f3c4f0d4-5a05-4719-9d33-b2f18e4083c8 kind: Event lastTimestamp: "2026-04-16T18:31:28Z" message: InferenceGraph [ensemble-graph-bac14] is Ready metadata: creationTimestamp: "2026-04-16T18:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:31:28Z" name: ensemble-graph-bac14.18a6e9e89973bd04 namespace: kserve-ci-e2e-test resourceVersion: "20717" uid: 70eb1b7a-272c-4532-8247-5fcbf9340e3d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-bac14 namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: a606ee95-1aa0-403c-9292-945c6f4d6e13 kind: Event lastTimestamp: "2026-04-16T18:31:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:43Z" name: ensemble-graph-bac14.18a6e9ec0f84d039 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 2865a022-8587-4872-af50-f49dbb1ab210 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-bac14 namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: a606ee95-1aa0-403c-9292-945c6f4d6e13 kind: Event lastTimestamp: "2026-04-16T18:31:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:43Z" name: ensemble-graph-bac14.18a6e9ec0ffc126c namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: eeba3189-1f92-416f-950f-bb2d0a02bddf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-bac14 namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: a606ee95-1aa0-403c-9292-945c6f4d6e13 kind: Event lastTimestamp: "2026-04-16T18:31:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:31:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:58Z" name: ensemble-graph-bac14.18a6e9ef8ebccf92 namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: 6a9f92d9-1a48-4ec7-883d-55a6c1509091 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-bac14 namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: a606ee95-1aa0-403c-9292-945c6f4d6e13 kind: Event lastTimestamp: "2026-04-16T18:31:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:31:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:58Z" name: ensemble-graph-bac14.18a6e9ef8f137639 namespace: kserve-ci-e2e-test resourceVersion: "20969" uid: c05ee873-a474-4bba-9074-a7ab7694069c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:30:07.710403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cc45f-cb59667b-gd9bj namespace: kserve-ci-e2e-test resourceVersion: "19679" uid: 510fb515-99fa-4b73-b49c-227f18031b76 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-cc45f-cb59667b-gd9bj to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:30:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:30:07Z" name: ensemble-graph-cc45f-cb59667b-gd9bj.18a6e9d5d84c09c3 namespace: kserve-ci-e2e-test resourceVersion: "19686" uid: a02a5757-96ca-40c2-aa4b-062316da5615 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:08Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cc45f-cb59667b-gd9bj namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 510fb515-99fa-4b73-b49c-227f18031b76 kind: Event lastTimestamp: "2026-04-16T18:30:08Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:30:08Z" name: ensemble-graph-cc45f-cb59667b-gd9bj.18a6e9d5f16fd1d5 namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: 446259c5-6915-43df-8e37-b6a310c76f58 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cc45f} kind: Pod name: ensemble-graph-cc45f-cb59667b-gd9bj namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 510fb515-99fa-4b73-b49c-227f18031b76 kind: Event lastTimestamp: "2026-04-16T18:30:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:08Z" name: ensemble-graph-cc45f-cb59667b-gd9bj.18a6e9d5f325367d namespace: kserve-ci-e2e-test resourceVersion: "19702" uid: 9d37245f-2616-4cb8-9b66-cae931ccd457 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cc45f} kind: Pod name: ensemble-graph-cc45f-cb59667b-gd9bj namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 510fb515-99fa-4b73-b49c-227f18031b76 kind: Event lastTimestamp: "2026-04-16T18:30:08Z" message: 'Created container: ensemble-graph-cc45f' metadata: creationTimestamp: "2026-04-16T18:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:08Z" name: ensemble-graph-cc45f-cb59667b-gd9bj.18a6e9d5f5e2f814 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 1a113696-fb3d-486d-bf96-9b4bc8b6214e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cc45f} kind: Pod name: ensemble-graph-cc45f-cb59667b-gd9bj namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 510fb515-99fa-4b73-b49c-227f18031b76 kind: Event lastTimestamp: "2026-04-16T18:30:08Z" message: Started container ensemble-graph-cc45f metadata: creationTimestamp: "2026-04-16T18:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:08Z" name: ensemble-graph-cc45f-cb59667b-gd9bj.18a6e9d5f658a985 namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: a645b085-0432-42c8-9a84-d48f29f8bdf6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cc45f} kind: Pod name: ensemble-graph-cc45f-cb59667b-gd9bj namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 510fb515-99fa-4b73-b49c-227f18031b76 kind: Event lastTimestamp: "2026-04-16T18:30:17Z" message: Stopping container ensemble-graph-cc45f metadata: creationTimestamp: "2026-04-16T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:17Z" name: ensemble-graph-cc45f-cb59667b-gd9bj.18a6e9d8334ce847 namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: 680a2bca-d9f6-4155-9a37-622eecde3f68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:30:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cc45f} kind: Pod name: ensemble-graph-cc45f-cb59667b-gd9bj namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 510fb515-99fa-4b73-b49c-227f18031b76 kind: Event lastTimestamp: "2026-04-16T18:30:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:44Z" name: ensemble-graph-cc45f-cb59667b-gd9bj.18a6e9d8a26bedaf namespace: kserve-ci-e2e-test resourceVersion: "20160" uid: f335669b-b3d1-44fa-8fef-09a661d7d796 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-cc45f-cb59667b namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: 25fef89e-017d-444e-a089-bbf10d84297c kind: Event lastTimestamp: "2026-04-16T18:30:07Z" message: 'Created pod: ensemble-graph-cc45f-cb59667b-gd9bj' metadata: creationTimestamp: "2026-04-16T18:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:07Z" name: ensemble-graph-cc45f-cb59667b.18a6e9d5d78276c3 namespace: kserve-ci-e2e-test resourceVersion: "19681" uid: d57ad325-bdf9-4812-9d26-2161d7049f9e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-cc45f namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: 3681a46b-4d69-47b9-bff0-eec44595042b kind: Event lastTimestamp: "2026-04-16T18:30:07Z" message: Scaled up replica set ensemble-graph-cc45f-cb59667b from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:07Z" name: ensemble-graph-cc45f.18a6e9d5d64ac580 namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: aab04abf-63da-4408-a622-052f5ed24f90 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-cc45f namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: abce545d-04a4-4cf4-917f-4bfe70b819c9 kind: Event lastTimestamp: "2026-04-16T18:30:07Z" message: InferenceGraph [ensemble-graph-cc45f] is Ready metadata: creationTimestamp: "2026-04-16T18:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:07Z" name: ensemble-graph-cc45f.18a6e9d5dfde9abe namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 647b10cf-bb13-4dab-9a5c-5de3db7347c3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:19:32.627498Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf namespace: kserve-ci-e2e-test resourceVersion: "14505" uid: 27188baf-2587-4799-8e1f-8d66cbbef954 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf.18a6e941fa6806ef namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: e2db851d-3484-467c-944c-22dd96ade05a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 27188baf-2587-4799-8e1f-8d66cbbef954 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf.18a6e94202b5f5f4 namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: 3e703ae0-d9c4-4412-a246-ad570e521842 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: 27188baf-2587-4799-8e1f-8d66cbbef954 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf.18a6e94203d49e25 namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: 24df7ce6-4855-43e8-84b7-b29bee9a8672 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: 27188baf-2587-4799-8e1f-8d66cbbef954 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf.18a6e9420695eed8 namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: b437cfaf-4dd5-40fc-a998-4864c1df05ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: 27188baf-2587-4799-8e1f-8d66cbbef954 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf.18a6e942071b3b38 namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 780ce817-ffbd-4dd1-ae60-66dd8f7e082d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:19:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: 27188baf-2587-4799-8e1f-8d66cbbef954 kind: Event lastTimestamp: "2026-04-16T18:20:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:19:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:14Z" name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf.18a6e94242e4f7be namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 5b7efd58-b94f-4c6d-b2bc-7a88cea851d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: 27188baf-2587-4799-8e1f-8d66cbbef954 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf.18a6e9c584cc10b3 namespace: kserve-ci-e2e-test resourceVersion: "18935" uid: af0e26dd-ae69-4a56-8eb3-36b4f8a961ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-14a0c-predictor-78c89c94f9 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 25a59607-2e14-4185-a6f9-8007a1d6d628 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'Created pod: error-404-isvc-14a0c-predictor-78c89c94f9-wnmmf' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-14a0c-predictor-78c89c94f9.18a6e941fa18b60c namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 8b3f61e0-4d6c-4690-af62-146702959c1f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 828f5abe-83d2-466f-87ff-fc71535adeeb kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Scaled up replica set error-404-isvc-14a0c-predictor-78c89c94f9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-14a0c-predictor.18a6e941f8c48038 namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: 991e333e-29f4-40d6-9886-975ad76a9e8f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:19:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: b7ef6cd0-ade6-46b1-9a82-5f8b17390d87 kind: Event lastTimestamp: "2026-04-16T18:20:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:17Z" name: error-404-isvc-14a0c-predictor.18a6e9457758b133 namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: a48c24ab-1a1e-4bf9-b180-177dfc922b18 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:19:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: b7ef6cd0-ade6-46b1-9a82-5f8b17390d87 kind: Event lastTimestamp: "2026-04-16T18:20:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:17Z" name: error-404-isvc-14a0c-predictor.18a6e94577c33f39 namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: a1bbf05f-2544-45c5-80c4-b78e48745f36 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:20:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: b7ef6cd0-ade6-46b1-9a82-5f8b17390d87 kind: Event lastTimestamp: "2026-04-16T18:20:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:47Z" name: error-404-isvc-14a0c-predictor.18a6e94ff43706ec namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: 14301233-d264-4b3d-b22f-7c4241689568 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:20:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: b7ef6cd0-ade6-46b1-9a82-5f8b17390d87 kind: Event lastTimestamp: "2026-04-16T18:20:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:47Z" name: error-404-isvc-14a0c-predictor.18a6e94ff4a34f67 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: c91f41bf-2250-45fd-9f42-cdfcb380970d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14a0c namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 7d009d1d-fe8a-4d28-bce0-964700d50578 kind: Event lastTimestamp: "2026-04-16T18:20:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:20:24Z" name: error-404-isvc-14a0c.18a6e941f89a33c4 namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: e5c56d6b-bf0a-4010-8f1d-a15442529257 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14a0c namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 7d009d1d-fe8a-4d28-bce0-964700d50578 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'Failed to update status for InferenceService "error-404-isvc-14a0c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-14a0c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-14a0c.18a6e941f8e7afc1 namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: 3290c782-3227-4867-b556-ee6edf6dfb00 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14a0c namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 7d009d1d-fe8a-4d28-bce0-964700d50578 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-14a0c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-14a0c.18a6e941f8e810b8 namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: f4023288-5c56-4ccf-a1cc-7400fc594b43 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14a0c namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: 7d009d1d-fe8a-4d28-bce0-964700d50578 kind: Event lastTimestamp: "2026-04-16T18:20:24Z" message: InferenceService [error-404-isvc-14a0c] is Ready metadata: creationTimestamp: "2026-04-16T18:20:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:20:24Z" name: error-404-isvc-14a0c.18a6e94e28f22f34 namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: 959d54c0-0673-4915-bd20-f19b7caff790 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:41:03.378460Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p namespace: kserve-ci-e2e-test resourceVersion: "25225" uid: 7bbea509-f542-489a-85da-ec07e5fa4820 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p.18a6ea6e8128a2fe namespace: kserve-ci-e2e-test resourceVersion: "25229" uid: f94acd45-8f08-4b16-8c40-8323deaabe82 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p namespace: kserve-ci-e2e-test resourceVersion: "25232" uid: 7bbea509-f542-489a-85da-ec07e5fa4820 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p.18a6ea6e89b16997 namespace: kserve-ci-e2e-test resourceVersion: "25244" uid: ebc18717-1aed-421a-b0f2-ef928944d47c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p namespace: kserve-ci-e2e-test resourceVersion: "25228" uid: 7bbea509-f542-489a-85da-ec07e5fa4820 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p.18a6ea6e8af18b9f namespace: kserve-ci-e2e-test resourceVersion: "25247" uid: bbba9573-6b86-4f94-8222-ae6421506181 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p namespace: kserve-ci-e2e-test resourceVersion: "25228" uid: 7bbea509-f542-489a-85da-ec07e5fa4820 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p.18a6ea6e90734200 namespace: kserve-ci-e2e-test resourceVersion: "25254" uid: c8fbfeb3-7a9f-474d-b054-18a93d16c38e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p namespace: kserve-ci-e2e-test resourceVersion: "25228" uid: 7bbea509-f542-489a-85da-ec07e5fa4820 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p.18a6ea6e90ebc001 namespace: kserve-ci-e2e-test resourceVersion: "25255" uid: fa3b71ba-9589-43b9-949d-1c507ab47393 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p namespace: kserve-ci-e2e-test resourceVersion: "25228" uid: 7bbea509-f542-489a-85da-ec07e5fa4820 kind: Event lastTimestamp: "2026-04-16T18:41:45Z" message: 'Readiness probe failed: dial tcp 10.133.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:45Z" name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p.18a6ea6eab97ff6e namespace: kserve-ci-e2e-test resourceVersion: "25652" uid: b9bd9fe0-8242-4e7c-a9b8-4172afbbe4df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p namespace: kserve-ci-e2e-test resourceVersion: "25228" uid: 7bbea509-f542-489a-85da-ec07e5fa4820 kind: Event lastTimestamp: "2026-04-16T18:50:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:50:28Z" name: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p.18a6eaf203b73373 namespace: kserve-ci-e2e-test resourceVersion: "29669" uid: d2dc2164-163c-461b-87bc-b8aadc9d6f4b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1ac5a-predictor-697b8f655f namespace: kserve-ci-e2e-test resourceVersion: "25217" uid: f9307115-9a0e-4650-b344-51fdba18412c kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: 'Created pod: error-404-isvc-1ac5a-predictor-697b8f655f-5nh8p' metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-1ac5a-predictor-697b8f655f.18a6ea6e806c445c namespace: kserve-ci-e2e-test resourceVersion: "25227" uid: 5d42c3ed-8a3a-4ed8-af72-ef5170b7a080 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25216" uid: a09ff436-b02c-4850-9aca-6329fa7a0a54 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Scaled up replica set error-404-isvc-1ac5a-predictor-697b8f655f from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-1ac5a-predictor.18a6ea6e7eace448 namespace: kserve-ci-e2e-test resourceVersion: "25219" uid: 20fe8e79-9c32-42e0-9939-f614edc24608 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: f456280e-b900-4101-90c6-635590f76700 kind: Event lastTimestamp: "2026-04-16T18:41:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:18Z" name: error-404-isvc-1ac5a-predictor.18a6ea71fd877f15 namespace: kserve-ci-e2e-test resourceVersion: "25394" uid: 1d8dda91-d06f-47d7-9bfd-447563faa498 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: f456280e-b900-4101-90c6-635590f76700 kind: Event lastTimestamp: "2026-04-16T18:41:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:18Z" name: error-404-isvc-1ac5a-predictor.18a6ea71fe075bd5 namespace: kserve-ci-e2e-test resourceVersion: "25396" uid: 8d74f19c-64b2-4e72-80f7-6c5ea496c863 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:41:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: f456280e-b900-4101-90c6-635590f76700 kind: Event lastTimestamp: "2026-04-16T18:42:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:18Z" name: error-404-isvc-1ac5a-predictor.18a6ea757cbda72b namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: 76a2f0d3-99c6-4c9a-b30f-6db68c025f18 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:41:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: f456280e-b900-4101-90c6-635590f76700 kind: Event lastTimestamp: "2026-04-16T18:42:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:18Z" name: error-404-isvc-1ac5a-predictor.18a6ea757d228496 namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 99f5b4e6-3f8c-4058-a41e-32de2798c0c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25196" uid: 3e014e67-e03e-4f52-ba4b-d38fc45bb5c7 kind: Event lastTimestamp: "2026-04-16T18:41:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:55Z" name: error-404-isvc-1ac5a.18a6ea6e7ed498d6 namespace: kserve-ci-e2e-test resourceVersion: "25751" uid: f3a0da4f-9fda-4b4e-ab45-26a679b0c30c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25196" uid: 3e014e67-e03e-4f52-ba4b-d38fc45bb5c7 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-1ac5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ac5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-1ac5a.18a6ea6e7f5ce0e5 namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: a0561038-5f66-45b6-ae7c-125a9e201c6e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25196" uid: 3e014e67-e03e-4f52-ba4b-d38fc45bb5c7 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ac5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-1ac5a.18a6ea6e7f5d2d3f namespace: kserve-ci-e2e-test resourceVersion: "25224" uid: a4cc5e8d-aee2-4b69-89cb-4dbde0431525 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25746" uid: 3e014e67-e03e-4f52-ba4b-d38fc45bb5c7 kind: Event lastTimestamp: "2026-04-16T18:41:55Z" message: InferenceService [error-404-isvc-1ac5a] is Ready metadata: creationTimestamp: "2026-04-16T18:41:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:55Z" name: error-404-isvc-1ac5a.18a6ea7a8fc804c9 namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: fe3c5995-c7a4-4f27-98b9-91276dee4ffa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:18:02.503979Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch namespace: kserve-ci-e2e-test resourceVersion: "13511" uid: 390db12b-b4b7-4d09-836f-6ed116b1f1ad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-47091-predictor-5d9486d6d7-pwtch to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:18:02Z" name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch.18a6e92cfea04525 namespace: kserve-ci-e2e-test resourceVersion: "13520" uid: 43e95d20-5b79-490c-96e3-2178b658ee9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch namespace: kserve-ci-e2e-test resourceVersion: "13530" uid: 390db12b-b4b7-4d09-836f-6ed116b1f1ad kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:18:02Z" name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch.18a6e92d0653f284 namespace: kserve-ci-e2e-test resourceVersion: "13537" uid: 04ae39c0-ec97-4272-abee-da2647a6488a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch namespace: kserve-ci-e2e-test resourceVersion: "13515" uid: 390db12b-b4b7-4d09-836f-6ed116b1f1ad kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:02Z" name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch.18a6e92d07858ecf namespace: kserve-ci-e2e-test resourceVersion: "13541" uid: 250b6f2d-d5fe-425f-887d-311b4247956a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch namespace: kserve-ci-e2e-test resourceVersion: "13515" uid: 390db12b-b4b7-4d09-836f-6ed116b1f1ad kind: Event lastTimestamp: "2026-04-16T18:18:19Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" in 16.678s (16.678s including waiting). Image size: 1332159081 bytes.' metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:19Z" name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch.18a6e930e9a909ba namespace: kserve-ci-e2e-test resourceVersion: "13738" uid: ea3276de-108e-428b-9cce-8c90455e5c27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch namespace: kserve-ci-e2e-test resourceVersion: "13515" uid: 390db12b-b4b7-4d09-836f-6ed116b1f1ad kind: Event lastTimestamp: "2026-04-16T18:18:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:19Z" name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch.18a6e930ef96a681 namespace: kserve-ci-e2e-test resourceVersion: "13743" uid: f34db68b-89c3-4a2f-b4fd-0e09e4fa948b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch namespace: kserve-ci-e2e-test resourceVersion: "13515" uid: 390db12b-b4b7-4d09-836f-6ed116b1f1ad kind: Event lastTimestamp: "2026-04-16T18:18:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:19Z" name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch.18a6e930effd9163 namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: 782cf94d-c364-42d6-b464-9a3c4d44b4ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch namespace: kserve-ci-e2e-test resourceVersion: "13515" uid: 390db12b-b4b7-4d09-836f-6ed116b1f1ad kind: Event lastTimestamp: "2026-04-16T18:19:00Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:00Z" name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch.18a6e930f8fac963 namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: 1ce68b88-1d47-4b81-af0d-393ba1f32455 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch namespace: kserve-ci-e2e-test resourceVersion: "13515" uid: 390db12b-b4b7-4d09-836f-6ed116b1f1ad kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: error-404-isvc-47091-predictor-5d9486d6d7-pwtch.18a6e941f09f3691 namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 6798c9c7-3e70-42b6-b0b9-c9e6b47844ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-47091-predictor-5d9486d6d7 namespace: kserve-ci-e2e-test resourceVersion: "13500" uid: c3680f20-f388-47bb-93d9-40bec5c2cabf kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: 'Created pod: error-404-isvc-47091-predictor-5d9486d6d7-pwtch' metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: error-404-isvc-47091-predictor-5d9486d6d7.18a6e92cfe1aa807 namespace: kserve-ci-e2e-test resourceVersion: "13514" uid: e2621fd8-dd1c-45b1-8f32-766d94721265 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13497" uid: 5cabf8bf-1945-4581-85b9-9f90eaf4681f kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Scaled up replica set error-404-isvc-47091-predictor-5d9486d6d7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: error-404-isvc-47091-predictor.18a6e92cfac6e868 namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: 46679d54-20d9-494e-8bdb-c2c1fc4d5e05 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13499" uid: aae45446-fe38-4249-a422-adf884d55cfa kind: Event lastTimestamp: "2026-04-16T18:18:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:47Z" name: error-404-isvc-47091-predictor.18a6e93079829188 namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 77a97cf4-067d-4009-b26d-e625766e1242 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13499" uid: aae45446-fe38-4249-a422-adf884d55cfa kind: Event lastTimestamp: "2026-04-16T18:18:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:47Z" name: error-404-isvc-47091-predictor.18a6e93079f2529c namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: c60a581d-daf2-47f1-b19b-00cf6f0e7e62 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:19:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13717" uid: aae45446-fe38-4249-a422-adf884d55cfa kind: Event lastTimestamp: "2026-04-16T18:19:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:17Z" name: error-404-isvc-47091-predictor.18a6e93af6537108 namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 56f890c3-6a3e-4289-9bc0-b8019a49612d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:19:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13717" uid: aae45446-fe38-4249-a422-adf884d55cfa kind: Event lastTimestamp: "2026-04-16T18:19:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:17Z" name: error-404-isvc-47091-predictor.18a6e93af6bfe26f namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b18bb0f7-1424-4560-8a3f-21a4019abcb8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-47091 namespace: kserve-ci-e2e-test resourceVersion: "13483" uid: 673635dd-b299-4b62-88ce-b7ebf6882c15 kind: Event lastTimestamp: "2026-04-16T18:19:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:10Z" name: error-404-isvc-47091.18a6e92cfacbd6ca namespace: kserve-ci-e2e-test resourceVersion: "14201" uid: 09904a34-ded0-4bcc-9edd-5adbc5c63cbb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-47091 namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: 673635dd-b299-4b62-88ce-b7ebf6882c15 kind: Event lastTimestamp: "2026-04-16T18:19:10Z" message: InferenceService [error-404-isvc-47091] is Ready metadata: creationTimestamp: "2026-04-16T18:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:10Z" name: error-404-isvc-47091.18a6e93cdf98c284 namespace: kserve-ci-e2e-test resourceVersion: "14199" uid: ad49ef0a-ff26-432a-bea0-6516e76f34e9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:20:22.526536Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: fcd952a4-c57d-4b6d-be26-6984769def9a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-68d6a-predictor-84c8c878d7-tljnw to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:20:22Z" name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw.18a6e94d989ee159 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 4c2c56e6-7178-4804-95e9-45603a5172ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: fcd952a4-c57d-4b6d-be26-6984769def9a kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:20:22Z" name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw.18a6e94da0af674f namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: 85c3661b-8785-4fb0-a21d-34ad953bf9a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fcd952a4-c57d-4b6d-be26-6984769def9a kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw.18a6e94da1f36cce namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: a1ed90b1-36c9-4f22-92e3-405a24b888d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fcd952a4-c57d-4b6d-be26-6984769def9a kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw.18a6e94da5f428cc namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 5214b2f5-8388-41ab-9fe4-5d06486afab8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fcd952a4-c57d-4b6d-be26-6984769def9a kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw.18a6e94da6626ce4 namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: c6707fa4-8d87-4964-810b-59dbb48c1a24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:20:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fcd952a4-c57d-4b6d-be26-6984769def9a kind: Event lastTimestamp: "2026-04-16T18:21:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:20:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:21:04Z" name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw.18a6e94db59adeed namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: a14bf49f-f55a-4b74-8e97-9805e672476a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fcd952a4-c57d-4b6d-be26-6984769def9a kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw.18a6e9d126f72e30 namespace: kserve-ci-e2e-test resourceVersion: "19446" uid: 88118a70-8bb7-4702-b3c1-2df81c27f459 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-68d6a-predictor-84c8c878d7 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: bc36e1d1-a7dd-471b-83bc-679216c2b604 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: 'Created pod: error-404-isvc-68d6a-predictor-84c8c878d7-tljnw' metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:22Z" name: error-404-isvc-68d6a-predictor-84c8c878d7.18a6e94d982d20aa namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: baa59ad8-acb8-4cfa-9431-9f37a0ddddda reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 2437117c-1d8b-4592-8b82-dcec33750d35 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Scaled up replica set error-404-isvc-68d6a-predictor-84c8c878d7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:22Z" name: error-404-isvc-68d6a-predictor.18a6e94d96bdb9c1 namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: 2ea2be0e-e5d9-44a0-9cf5-6d86536afa37 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:20:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 79a909af-7394-4fd6-9c21-ed43bf5d4339 kind: Event lastTimestamp: "2026-04-16T18:21:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:07Z" name: error-404-isvc-68d6a-predictor.18a6e951154bd118 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: ee2aa70c-d572-44d9-a078-55caa4fc4662 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:20:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 79a909af-7394-4fd6-9c21-ed43bf5d4339 kind: Event lastTimestamp: "2026-04-16T18:21:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:07Z" name: error-404-isvc-68d6a-predictor.18a6e95115b025c1 namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: fe955ed0-8c4d-4425-8db6-74cd3ec818c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:21:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 79a909af-7394-4fd6-9c21-ed43bf5d4339 kind: Event lastTimestamp: "2026-04-16T18:21:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:21:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:37Z" name: error-404-isvc-68d6a-predictor.18a6e95b921189ab namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: cb72f5b3-8871-425a-8a03-9d9d881be166 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:21:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 79a909af-7394-4fd6-9c21-ed43bf5d4339 kind: Event lastTimestamp: "2026-04-16T18:21:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:21:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:37Z" name: error-404-isvc-68d6a-predictor.18a6e95b9279034c namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 02e2d76f-58a3-496a-bac2-8744cebc6e28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: e67d14ee-49c6-496c-8c00-5223c3199bad kind: Event lastTimestamp: "2026-04-16T18:21:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:21:14Z" name: error-404-isvc-68d6a.18a6e94d9696e710 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: fb8a342a-1d34-4c2d-bd17-aead7b67e75e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: e67d14ee-49c6-496c-8c00-5223c3199bad kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: 'Failed to update status for InferenceService "error-404-isvc-68d6a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-68d6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:20:22Z" name: error-404-isvc-68d6a.18a6e94d96e4acaf namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: d3055c9b-0022-4df4-aa6e-61e5bec25438 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: e67d14ee-49c6-496c-8c00-5223c3199bad kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-68d6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:20:22Z" name: error-404-isvc-68d6a.18a6e94d96e50c93 namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: 00d7b0ce-f9ef-44a2-ad72-784b84e06258 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: e67d14ee-49c6-496c-8c00-5223c3199bad kind: Event lastTimestamp: "2026-04-16T18:21:14Z" message: InferenceService [error-404-isvc-68d6a] is Ready metadata: creationTimestamp: "2026-04-16T18:21:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:21:14Z" name: error-404-isvc-68d6a.18a6e95999db9ce6 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 0744c469-1bc4-4c79-8160-53fcb9205d3f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:30:18.284546Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9 namespace: kserve-ci-e2e-test resourceVersion: "19891" uid: 7b9ab55c-1bd0-4f32-8561-b12fc9b20f77 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bac14-predictor-5465cc4b59-fkqw9 to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9.18a6e9d84e90af9a namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: 9dcc40b7-fb5f-41d0-998a-9507f4bb0c6f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9 namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 7b9ab55c-1bd0-4f32-8561-b12fc9b20f77 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9.18a6e9d8561d0caf namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: 48fd0d94-2a45-4372-8f05-ad2fbecb82f7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9 namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 7b9ab55c-1bd0-4f32-8561-b12fc9b20f77 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9.18a6e9d858caa35f namespace: kserve-ci-e2e-test resourceVersion: "19913" uid: fadc1200-ad88-4d9f-a92b-61aeaea28530 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9 namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 7b9ab55c-1bd0-4f32-8561-b12fc9b20f77 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9.18a6e9d85d7d523b namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 631459e4-559c-44df-880d-936c8e94ccbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9 namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 7b9ab55c-1bd0-4f32-8561-b12fc9b20f77 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9.18a6e9d85dee714e namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: c7a752fe-80d7-4c79-9c94-2e7ba07c94b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9 namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 7b9ab55c-1bd0-4f32-8561-b12fc9b20f77 kind: Event lastTimestamp: "2026-04-16T18:30:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:59Z" name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9.18a6e9d8770dcfc0 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 47095adc-0d00-4ac4-933f-f26583095812 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9 namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 7b9ab55c-1bd0-4f32-8561-b12fc9b20f77 kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:42Z" name: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9.18a6ea5bc385eada namespace: kserve-ci-e2e-test resourceVersion: "24273" uid: 6754857f-7ee4-4375-a5e5-8e068bd21eb1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bac14-predictor-5465cc4b59 namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 9ef5dc6e-e679-429d-89c9-2d0beaa512f3 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: 'Created pod: error-404-isvc-bac14-predictor-5465cc4b59-fkqw9' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-bac14-predictor-5465cc4b59.18a6e9d84cee480a namespace: kserve-ci-e2e-test resourceVersion: "19894" uid: bee8cc77-d641-48a3-bc6b-1011f3d9a6f7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: e405d517-2b61-4414-83c1-97097072b347 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: Scaled up replica set error-404-isvc-bac14-predictor-5465cc4b59 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-bac14-predictor.18a6e9d8485047ba namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 6d25b196-6c10-401c-8d24-ea899def5cea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: 901933af-ab56-4826-88c4-d7f7e50c38b4 kind: Event lastTimestamp: "2026-04-16T18:30:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:33Z" name: error-404-isvc-bac14-predictor.18a6e9dbc77f0371 namespace: kserve-ci-e2e-test resourceVersion: "20053" uid: 645287b8-96c4-4feb-835d-31567d10ace5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: 901933af-ab56-4826-88c4-d7f7e50c38b4 kind: Event lastTimestamp: "2026-04-16T18:30:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:33Z" name: error-404-isvc-bac14-predictor.18a6e9dbc860b20b namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 2f4ffcd1-3e76-4422-b357-1cebef5aaba4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:30:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 901933af-ab56-4826-88c4-d7f7e50c38b4 kind: Event lastTimestamp: "2026-04-16T18:31:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:33Z" name: error-404-isvc-bac14-predictor.18a6e9df475cf973 namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: da2ee20a-3766-4648-baae-355bfb816354 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:30:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 901933af-ab56-4826-88c4-d7f7e50c38b4 kind: Event lastTimestamp: "2026-04-16T18:31:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:33Z" name: error-404-isvc-bac14-predictor.18a6e9df47d699f5 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 5f5ea4b0-e4d4-466b-af48-46a5c9e9c61a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bac14 namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: be7cbf60-31f3-4066-995a-e0f8b295acb9 kind: Event lastTimestamp: "2026-04-16T18:31:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:31:10Z" name: error-404-isvc-bac14.18a6e9d848453176 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 950d3838-ae44-4a2b-89c1-3fd8872fcfdc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bac14 namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: be7cbf60-31f3-4066-995a-e0f8b295acb9 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: 'Failed to update status for InferenceService "error-404-isvc-bac14": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bac14": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-bac14.18a6e9d84905290e namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: ab5bb3fe-53fd-44d5-b924-f59d51efb579 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bac14 namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: be7cbf60-31f3-4066-995a-e0f8b295acb9 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bac14": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-bac14.18a6e9d849058ce3 namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: c1987bc5-9303-4f5d-ba01-84381f9be62f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bac14 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: be7cbf60-31f3-4066-995a-e0f8b295acb9 kind: Event lastTimestamp: "2026-04-16T18:31:10Z" message: InferenceService [error-404-isvc-bac14] is Ready metadata: creationTimestamp: "2026-04-16T18:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:31:10Z" name: error-404-isvc-bac14.18a6e9e463f115cf namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: f8f56dd5-13dd-4848-be54-22664f37cc0c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:29:47.586500Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: f3470a1d-471d-46f5-bfa1-174b9c19e40d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c659a-predictor-869ffd459c-2g2gh to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh.18a6e9d128d1b34a namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 3f068266-6183-48e4-ab13-8862dda06b9c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh namespace: kserve-ci-e2e-test resourceVersion: "19458" uid: f3470a1d-471d-46f5-bfa1-174b9c19e40d kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh.18a6e9d130ceb05e namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: 34ca5949-92ec-40ae-8eec-dca14672e990 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: f3470a1d-471d-46f5-bfa1-174b9c19e40d kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh.18a6e9d13219bf86 namespace: kserve-ci-e2e-test resourceVersion: "19471" uid: 2812e7ab-f184-4d17-bf28-971919a6b4ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: f3470a1d-471d-46f5-bfa1-174b9c19e40d kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh.18a6e9d1379506e8 namespace: kserve-ci-e2e-test resourceVersion: "19475" uid: f6a565ef-458c-4918-95b4-8f99f52ba07f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: f3470a1d-471d-46f5-bfa1-174b9c19e40d kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh.18a6e9d1381b96ea namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: 405ac260-c230-4686-a8e3-863f682bd5be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: f3470a1d-471d-46f5-bfa1-174b9c19e40d kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh.18a6e9d138da7667 namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: 0a65dc81-74c2-494a-812c-120381cf796b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: f3470a1d-471d-46f5-bfa1-174b9c19e40d kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: error-404-isvc-c659a-predictor-869ffd459c-2g2gh.18a6e9e3ff094811 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 1b15de97-837f-4ef9-8270-5150943683da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c659a-predictor-869ffd459c namespace: kserve-ci-e2e-test resourceVersion: "19438" uid: 7ddfcba3-1949-4d6b-b1d2-6e1dfa7b8f46 kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: 'Created pod: error-404-isvc-c659a-predictor-869ffd459c-2g2gh' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-c659a-predictor-869ffd459c.18a6e9d12818257c namespace: kserve-ci-e2e-test resourceVersion: "19449" uid: 143e2874-2e79-4e3b-9648-e1ab01694a19 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19431" uid: 6bd3b3c3-fdc3-4bae-90d7-1b67c663cbe5 kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Scaled up replica set error-404-isvc-c659a-predictor-869ffd459c from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-c659a-predictor.18a6e9d1260aaef7 namespace: kserve-ci-e2e-test resourceVersion: "19441" uid: 7b6087ed-c898-4253-bcbb-42b60490a906 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19437" uid: 2eaff2ec-23ec-44d9-a472-e35d778607a4 kind: Event lastTimestamp: "2026-04-16T18:30:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:02Z" name: error-404-isvc-c659a-predictor.18a6e9d4a4f1b9dd namespace: kserve-ci-e2e-test resourceVersion: "19627" uid: 73e8baf0-30d5-40a4-a2f3-65cf8cd70f91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19437" uid: 2eaff2ec-23ec-44d9-a472-e35d778607a4 kind: Event lastTimestamp: "2026-04-16T18:30:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:02Z" name: error-404-isvc-c659a-predictor.18a6e9d4a5999d89 namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: 1bd2bc59-e901-4ca9-8d37-476ad7038120 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:30:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19626" uid: 2eaff2ec-23ec-44d9-a472-e35d778607a4 kind: Event lastTimestamp: "2026-04-16T18:31:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:02Z" name: error-404-isvc-c659a-predictor.18a6e9d824839d8a namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: d89986f4-774d-438c-9174-ee2c526e49af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:30:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19626" uid: 2eaff2ec-23ec-44d9-a472-e35d778607a4 kind: Event lastTimestamp: "2026-04-16T18:31:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:02Z" name: error-404-isvc-c659a-predictor.18a6e9d825058991 namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 8be18b84-451a-49e1-9f83-aeade18bfa6c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c659a namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 5f5aa782-ec29-4f1e-99d0-6618ce25dd4c kind: Event lastTimestamp: "2026-04-16T18:30:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:39Z" name: error-404-isvc-c659a.18a6e9d12612eb25 namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: 870ab01d-86ee-4be8-b222-4a4f37f0cd93 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c659a namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 5f5aa782-ec29-4f1e-99d0-6618ce25dd4c kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-c659a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c659a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-c659a.18a6e9d126bf737d namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: b529a77f-b968-4552-922d-67311ea8dc2b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c659a namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 5f5aa782-ec29-4f1e-99d0-6618ce25dd4c kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c659a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:47Z" name: error-404-isvc-c659a.18a6e9d126bfb221 namespace: kserve-ci-e2e-test resourceVersion: "19447" uid: 4b1e2336-da24-49b8-b526-7c0339175614 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c659a namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 5f5aa782-ec29-4f1e-99d0-6618ce25dd4c kind: Event lastTimestamp: "2026-04-16T18:30:39Z" message: InferenceService [error-404-isvc-c659a] is Ready metadata: creationTimestamp: "2026-04-16T18:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:39Z" name: error-404-isvc-c659a.18a6e9dd23f0d7ef namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: f15763ab-5caa-48f4-b1f6-d98c97d77d24 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:57.635658Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cc45f-predictor-c5c55775-f5627 namespace: kserve-ci-e2e-test resourceVersion: "18948" uid: 620cdff5-ebb8-4fce-9e06-726bda85841e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cc45f-predictor-c5c55775-f5627 to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-cc45f-predictor-c5c55775-f5627.18a6e9c5878464dc namespace: kserve-ci-e2e-test resourceVersion: "18952" uid: aa80758f-e292-4f66-98c3-84ab4e43600a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cc45f-predictor-c5c55775-f5627 namespace: kserve-ci-e2e-test resourceVersion: "18957" uid: 620cdff5-ebb8-4fce-9e06-726bda85841e kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-cc45f-predictor-c5c55775-f5627.18a6e9c58ff68e35 namespace: kserve-ci-e2e-test resourceVersion: "18964" uid: 7b7b2699-9c28-404d-9fb2-2bd906128cec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc45f-predictor-c5c55775-f5627 namespace: kserve-ci-e2e-test resourceVersion: "18950" uid: 620cdff5-ebb8-4fce-9e06-726bda85841e kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-cc45f-predictor-c5c55775-f5627.18a6e9c59129c9ed namespace: kserve-ci-e2e-test resourceVersion: "18967" uid: 55cb181d-0201-4f81-a68d-e9abe2e75872 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc45f-predictor-c5c55775-f5627 namespace: kserve-ci-e2e-test resourceVersion: "18950" uid: 620cdff5-ebb8-4fce-9e06-726bda85841e kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-cc45f-predictor-c5c55775-f5627.18a6e9c599471d34 namespace: kserve-ci-e2e-test resourceVersion: "18973" uid: 8360308f-1abb-480a-9c6c-cea5af6320f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc45f-predictor-c5c55775-f5627 namespace: kserve-ci-e2e-test resourceVersion: "18950" uid: 620cdff5-ebb8-4fce-9e06-726bda85841e kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-cc45f-predictor-c5c55775-f5627.18a6e9c599ac5666 namespace: kserve-ci-e2e-test resourceVersion: "18974" uid: b67e1e5f-1d1e-4687-b845-3a844536cd29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc45f-predictor-c5c55775-f5627 namespace: kserve-ci-e2e-test resourceVersion: "18950" uid: 620cdff5-ebb8-4fce-9e06-726bda85841e kind: Event lastTimestamp: "2026-04-16T18:30:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:19Z" name: error-404-isvc-cc45f-predictor-c5c55775-f5627.18a6e9c5c5df3b01 namespace: kserve-ci-e2e-test resourceVersion: "19939" uid: 756f2318-5d7a-4d69-9fa2-8589068d9424 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc45f-predictor-c5c55775-f5627 namespace: kserve-ci-e2e-test resourceVersion: "18950" uid: 620cdff5-ebb8-4fce-9e06-726bda85841e kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:18Z" name: error-404-isvc-cc45f-predictor-c5c55775-f5627.18a6e9d84266048a namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 21cf0321-4bfc-4e4a-ae47-3b4cce002db0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cc45f-predictor-c5c55775 namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: d650cc92-9cca-4eb2-aab9-75c3e83a846c kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: 'Created pod: error-404-isvc-cc45f-predictor-c5c55775-f5627' metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-cc45f-predictor-c5c55775.18a6e9c58719707e namespace: kserve-ci-e2e-test resourceVersion: "18949" uid: f6bb2316-fbc9-495a-8287-93849e25101e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18934" uid: ea2eb48d-90d2-4c38-89ce-960ffe7b9f57 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Scaled up replica set error-404-isvc-cc45f-predictor-c5c55775 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-cc45f-predictor.18a6e9c585926ed1 namespace: kserve-ci-e2e-test resourceVersion: "18940" uid: e20cb083-5134-496e-936a-ad072fe0546c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:29:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18939" uid: b86bcc04-0781-4f2d-8dfb-40ff76dbd38f kind: Event lastTimestamp: "2026-04-16T18:29:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:27Z" name: error-404-isvc-cc45f-predictor.18a6e9c90452acc1 namespace: kserve-ci-e2e-test resourceVersion: "19215" uid: c6d416e3-e080-489e-82c1-ba5982fe78bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:29:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18939" uid: b86bcc04-0781-4f2d-8dfb-40ff76dbd38f kind: Event lastTimestamp: "2026-04-16T18:29:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:27Z" name: error-404-isvc-cc45f-predictor.18a6e9c904c67639 namespace: kserve-ci-e2e-test resourceVersion: "19216" uid: da8154cb-2dc7-4876-a14c-fe5e4e21dcff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:29:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: b86bcc04-0781-4f2d-8dfb-40ff76dbd38f kind: Event lastTimestamp: "2026-04-16T18:30:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:12Z" name: error-404-isvc-cc45f-predictor.18a6e9d002feb212 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: b8bf95c1-bb5e-420a-8947-71224b5437ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:29:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: b86bcc04-0781-4f2d-8dfb-40ff76dbd38f kind: Event lastTimestamp: "2026-04-16T18:30:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:12Z" name: error-404-isvc-cc45f-predictor.18a6e9d003817405 namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: 0b4eba30-bd3e-429e-ab19-bc360a05712b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cc45f namespace: kserve-ci-e2e-test resourceVersion: "18910" uid: 3629eb84-4249-4430-bcb8-1fe708540d2f kind: Event lastTimestamp: "2026-04-16T18:29:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:49Z" name: error-404-isvc-cc45f.18a6e9c5859a3480 namespace: kserve-ci-e2e-test resourceVersion: "19519" uid: ed700462-f3d2-453d-a2f3-0b8d68287ea2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cc45f namespace: kserve-ci-e2e-test resourceVersion: "18910" uid: 3629eb84-4249-4430-bcb8-1fe708540d2f kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: 'Failed to update status for InferenceService "error-404-isvc-cc45f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cc45f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-cc45f.18a6e9c5860c3947 namespace: kserve-ci-e2e-test resourceVersion: "18944" uid: 442adbce-1a4d-43eb-b3ac-31bdbfb171de reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cc45f namespace: kserve-ci-e2e-test resourceVersion: "18910" uid: 3629eb84-4249-4430-bcb8-1fe708540d2f kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cc45f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:57Z" name: error-404-isvc-cc45f.18a6e9c5860c8ef9 namespace: kserve-ci-e2e-test resourceVersion: "18945" uid: 747f0dc7-68b8-44b7-9484-4949c77cba1c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cc45f namespace: kserve-ci-e2e-test resourceVersion: "19513" uid: 3629eb84-4249-4430-bcb8-1fe708540d2f kind: Event lastTimestamp: "2026-04-16T18:29:49Z" message: InferenceService [error-404-isvc-cc45f] is Ready metadata: creationTimestamp: "2026-04-16T18:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:49Z" name: error-404-isvc-cc45f.18a6e9d1aab1a2f8 namespace: kserve-ci-e2e-test resourceVersion: "19514" uid: 4aa4188e-1a5c-405c-a3f7-9b83267b9a8b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:40:32.816431Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6 namespace: kserve-ci-e2e-test resourceVersion: "24787" uid: 76f4850c-8d86-4234-926d-3f0f9b821487 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6 to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:40:32Z" name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6.18a6ea6763851b24 namespace: kserve-ci-e2e-test resourceVersion: "24791" uid: e7de9401-2ab6-4342-95f9-98a1b7d7306d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6 namespace: kserve-ci-e2e-test resourceVersion: "24796" uid: 76f4850c-8d86-4234-926d-3f0f9b821487 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:40:32Z" name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6.18a6ea676c209f25 namespace: kserve-ci-e2e-test resourceVersion: "24808" uid: ff2b8ffe-77be-4fda-bacd-801e483b9a78 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6 namespace: kserve-ci-e2e-test resourceVersion: "24789" uid: 76f4850c-8d86-4234-926d-3f0f9b821487 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:32Z" name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6.18a6ea676d9d6d37 namespace: kserve-ci-e2e-test resourceVersion: "24812" uid: fd39ee11-d52b-4b5e-9faf-b6778ce4505b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6 namespace: kserve-ci-e2e-test resourceVersion: "24789" uid: 76f4850c-8d86-4234-926d-3f0f9b821487 kind: Event lastTimestamp: "2026-04-16T18:40:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:33Z" name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6.18a6ea6770871792 namespace: kserve-ci-e2e-test resourceVersion: "24816" uid: c1a7443c-9310-450f-b6e3-a4e94d6070dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6 namespace: kserve-ci-e2e-test resourceVersion: "24789" uid: 76f4850c-8d86-4234-926d-3f0f9b821487 kind: Event lastTimestamp: "2026-04-16T18:40:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:33Z" name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6.18a6ea6770f81c70 namespace: kserve-ci-e2e-test resourceVersion: "24817" uid: dc62086b-50c4-46e6-9feb-1a1f971ab7a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6 namespace: kserve-ci-e2e-test resourceVersion: "24789" uid: 76f4850c-8d86-4234-926d-3f0f9b821487 kind: Event lastTimestamp: "2026-04-16T18:41:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:15Z" name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6.18a6ea67e525204c namespace: kserve-ci-e2e-test resourceVersion: "25367" uid: 42697603-06ce-4df0-9eb5-a2f1f482129c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6 namespace: kserve-ci-e2e-test resourceVersion: "24789" uid: 76f4850c-8d86-4234-926d-3f0f9b821487 kind: Event lastTimestamp: "2026-04-16T18:58:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:58:02Z" name: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6.18a6eb5bc9310852 namespace: kserve-ci-e2e-test resourceVersion: "32818" uid: 03bc6f32-b88d-4534-a07d-3724ff38bfc2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dbc56-predictor-7cc999bf45 namespace: kserve-ci-e2e-test resourceVersion: "24780" uid: 4c739f66-2d35-4d58-8b48-9af3b8ec4112 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: 'Created pod: error-404-isvc-dbc56-predictor-7cc999bf45-7bxj6' metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:32Z" name: error-404-isvc-dbc56-predictor-7cc999bf45.18a6ea676310cfa6 namespace: kserve-ci-e2e-test resourceVersion: "24788" uid: 9b088996-af3a-4e9b-9146-5481c77235ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24778" uid: fbab1e60-367e-41cb-9749-cea81d0b3685 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Scaled up replica set error-404-isvc-dbc56-predictor-7cc999bf45 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:32Z" name: error-404-isvc-dbc56-predictor.18a6ea6761d537ad namespace: kserve-ci-e2e-test resourceVersion: "24781" uid: 304dcab3-5d9f-4bd1-944d-0948c775bc40 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:40:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24779" uid: c9b43fa2-a373-446f-a4cb-ac7456e2a17d kind: Event lastTimestamp: "2026-04-16T18:41:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:02Z" name: error-404-isvc-dbc56-predictor.18a6ea6ae0874070 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 271c1d8a-f9b7-412a-a1ed-f6155c3b3850 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:40:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24779" uid: c9b43fa2-a373-446f-a4cb-ac7456e2a17d kind: Event lastTimestamp: "2026-04-16T18:41:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:02Z" name: error-404-isvc-dbc56-predictor.18a6ea6ae0f0597a namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: 133cacc6-cba6-4015-b06e-22cdba8e8eab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:41:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: c9b43fa2-a373-446f-a4cb-ac7456e2a17d kind: Event lastTimestamp: "2026-04-16T18:41:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:47Z" name: error-404-isvc-dbc56-predictor.18a6ea71de98b3f4 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: c66c225b-cc81-48c5-9efa-f7935c1d0d04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:41:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: c9b43fa2-a373-446f-a4cb-ac7456e2a17d kind: Event lastTimestamp: "2026-04-16T18:41:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:47Z" name: error-404-isvc-dbc56-predictor.18a6ea71df0a04d2 namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: dcd24fd5-f64f-4b88-99ee-f4cec7ab6ad1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "24769" uid: c4ebcd8d-c117-4198-843b-906426daf213 kind: Event lastTimestamp: "2026-04-16T18:41:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:26Z" name: error-404-isvc-dbc56.18a6ea6761de9a12 namespace: kserve-ci-e2e-test resourceVersion: "25469" uid: 4a826b80-17e2-4d06-8e46-642566db8d14 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "24769" uid: c4ebcd8d-c117-4198-843b-906426daf213 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: 'Failed to update status for InferenceService "error-404-isvc-dbc56": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dbc56": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:32Z" name: error-404-isvc-dbc56.18a6ea676267b3a0 namespace: kserve-ci-e2e-test resourceVersion: "24784" uid: e1900783-39b5-4b80-8815-f100d3234556 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "24769" uid: c4ebcd8d-c117-4198-843b-906426daf213 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dbc56": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:32Z" name: error-404-isvc-dbc56.18a6ea67626812a6 namespace: kserve-ci-e2e-test resourceVersion: "24786" uid: da549d5d-26d0-45b3-a262-432664c32222 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "25466" uid: c4ebcd8d-c117-4198-843b-906426daf213 kind: Event lastTimestamp: "2026-04-16T18:41:26Z" message: InferenceService [error-404-isvc-dbc56] is Ready metadata: creationTimestamp: "2026-04-16T18:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:26Z" name: error-404-isvc-dbc56.18a6ea73c98d3418 namespace: kserve-ci-e2e-test resourceVersion: "25468" uid: 2cadd5e7-23f3-4430-b679-c6e07d631734 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:39:42.918990Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8 namespace: kserve-ci-e2e-test resourceVersion: "24279" uid: cc9a9ac1-318b-438b-8ab5-13cafbc3a4d4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8 to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:39:42Z" name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8.18a6ea5bc56690c1 namespace: kserve-ci-e2e-test resourceVersion: "24284" uid: 10f51039-1ae0-4770-ae9d-2a828524aac7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8 namespace: kserve-ci-e2e-test resourceVersion: "24287" uid: cc9a9ac1-318b-438b-8ab5-13cafbc3a4d4 kind: Event lastTimestamp: "2026-04-16T18:39:43Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:39:43Z" name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8.18a6ea5bcdc06249 namespace: kserve-ci-e2e-test resourceVersion: "24299" uid: 4f548b81-1a2f-4c1f-94fb-54bbe0cd3da4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8 namespace: kserve-ci-e2e-test resourceVersion: "24282" uid: cc9a9ac1-318b-438b-8ab5-13cafbc3a4d4 kind: Event lastTimestamp: "2026-04-16T18:39:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:43Z" name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8.18a6ea5bcf16ebaa namespace: kserve-ci-e2e-test resourceVersion: "24301" uid: eca89dd6-7c2f-4078-b988-7edb3f22fe75 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8 namespace: kserve-ci-e2e-test resourceVersion: "24282" uid: cc9a9ac1-318b-438b-8ab5-13cafbc3a4d4 kind: Event lastTimestamp: "2026-04-16T18:39:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:43Z" name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8.18a6ea5bd20997d3 namespace: kserve-ci-e2e-test resourceVersion: "24304" uid: 565836dd-eda6-4d8a-a30c-94c03da26707 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8 namespace: kserve-ci-e2e-test resourceVersion: "24282" uid: cc9a9ac1-318b-438b-8ab5-13cafbc3a4d4 kind: Event lastTimestamp: "2026-04-16T18:39:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:43Z" name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8.18a6ea5bd28390eb namespace: kserve-ci-e2e-test resourceVersion: "24306" uid: 5334f474-248f-4731-837d-15b4eebcc5da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8 namespace: kserve-ci-e2e-test resourceVersion: "24282" uid: cc9a9ac1-318b-438b-8ab5-13cafbc3a4d4 kind: Event lastTimestamp: "2026-04-16T18:41:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:04Z" name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8.18a6ea5bfa53e052 namespace: kserve-ci-e2e-test resourceVersion: "25275" uid: 5dc1bd7f-3e4b-4ee9-a07e-8c1b8bd744d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8 namespace: kserve-ci-e2e-test resourceVersion: "24282" uid: cc9a9ac1-318b-438b-8ab5-13cafbc3a4d4 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:03Z" name: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8.18a6ea6e75fbd83e namespace: kserve-ci-e2e-test resourceVersion: "25176" uid: df10d4e9-eee0-4eae-a3fc-f58120444db6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e04f1-predictor-6cc8df4c7c namespace: kserve-ci-e2e-test resourceVersion: "24269" uid: f0eddc2f-5cb3-4043-8ab0-1684c7c3da98 kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: 'Created pod: error-404-isvc-e04f1-predictor-6cc8df4c7c-blqf8' metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:42Z" name: error-404-isvc-e04f1-predictor-6cc8df4c7c.18a6ea5bc4ec445c namespace: kserve-ci-e2e-test resourceVersion: "24281" uid: c40631a9-463b-46fd-8c6b-434a45749cdf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24264" uid: 2c6b21d0-32dd-49a9-9fee-66f9d37334a9 kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: Scaled up replica set error-404-isvc-e04f1-predictor-6cc8df4c7c from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:42Z" name: error-404-isvc-e04f1-predictor.18a6ea5bc386a34f namespace: kserve-ci-e2e-test resourceVersion: "24272" uid: 57629b5d-69f9-4e7c-8f4a-9d8b8604b4ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:39:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24268" uid: 5764543d-396c-4d6e-bdfa-ba442299cd28 kind: Event lastTimestamp: "2026-04-16T18:40:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:12Z" name: error-404-isvc-e04f1-predictor.18a6ea5f42402db8 namespace: kserve-ci-e2e-test resourceVersion: "24552" uid: a5c124e6-d776-477f-adcd-90d5705e5b78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:39:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24268" uid: 5764543d-396c-4d6e-bdfa-ba442299cd28 kind: Event lastTimestamp: "2026-04-16T18:40:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:12Z" name: error-404-isvc-e04f1-predictor.18a6ea5f42a13104 namespace: kserve-ci-e2e-test resourceVersion: "24553" uid: f007c859-9189-4f2e-b888-3793687e27fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:40:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24440" uid: 5764543d-396c-4d6e-bdfa-ba442299cd28 kind: Event lastTimestamp: "2026-04-16T18:40:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:57Z" name: error-404-isvc-e04f1-predictor.18a6ea6640256ec3 namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: 00240e6b-7c4a-454c-a4d3-cc3e5b8e59ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:40:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24440" uid: 5764543d-396c-4d6e-bdfa-ba442299cd28 kind: Event lastTimestamp: "2026-04-16T18:40:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:57Z" name: error-404-isvc-e04f1-predictor.18a6ea66408e882f namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 96a8feda-dce9-4723-ae1a-d1cd9b370ac0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "24243" uid: b527dfe5-46a3-41d3-9a9a-cc0dd77ef8a9 kind: Event lastTimestamp: "2026-04-16T18:40:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:34Z" name: error-404-isvc-e04f1.18a6ea5bc38d7ce6 namespace: kserve-ci-e2e-test resourceVersion: "24856" uid: a6624631-59f4-4bd3-967b-10d11a815a68 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "24243" uid: b527dfe5-46a3-41d3-9a9a-cc0dd77ef8a9 kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: 'Failed to update status for InferenceService "error-404-isvc-e04f1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e04f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:42Z" name: error-404-isvc-e04f1.18a6ea5bc419514d namespace: kserve-ci-e2e-test resourceVersion: "24277" uid: 7081cece-054c-47ca-beab-b95c987ae93c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "24243" uid: b527dfe5-46a3-41d3-9a9a-cc0dd77ef8a9 kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e04f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:42Z" name: error-404-isvc-e04f1.18a6ea5bc4199bfa namespace: kserve-ci-e2e-test resourceVersion: "24280" uid: 8485e99a-f93d-422c-822e-8cf8788cd695 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "24851" uid: b527dfe5-46a3-41d3-9a9a-cc0dd77ef8a9 kind: Event lastTimestamp: "2026-04-16T18:40:34Z" message: InferenceService [error-404-isvc-e04f1] is Ready metadata: creationTimestamp: "2026-04-16T18:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:34Z" name: error-404-isvc-e04f1.18a6ea67de9489a6 namespace: kserve-ci-e2e-test resourceVersion: "24852" uid: f195357f-fcfd-4528-a61f-50ca9015543d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:31:08.456835Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 4278d34b-1456-4b6a-b349-dc07785eda2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e9680-predictor-8689cf6cdf-k7trg to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:31:08Z" name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg.18a6e9e3fd110dd9 namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: 56e16d08-cf50-448c-a4db-f397fcc51d0b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: 4278d34b-1456-4b6a-b349-dc07785eda2f kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:31:08Z" name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg.18a6e9e410381ee1 namespace: kserve-ci-e2e-test resourceVersion: "20501" uid: c8e7f5c4-f2f9-41ea-a32e-05faf7822c75 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: 4278d34b-1456-4b6a-b349-dc07785eda2f kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg.18a6e9e413336c76 namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: e811b6fd-32d4-49a9-adbc-2547b1c03034 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: 4278d34b-1456-4b6a-b349-dc07785eda2f kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg.18a6e9e41961ec08 namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: f71d246b-911f-4df4-be2c-d952b3c470d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: 4278d34b-1456-4b6a-b349-dc07785eda2f kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg.18a6e9e419dbbd5e namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 7e53a956-76be-4013-8448-1fcea9d6a3bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: 4278d34b-1456-4b6a-b349-dc07785eda2f kind: Event lastTimestamp: "2026-04-16T18:31:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:50Z" name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg.18a6e9e4258faf9c namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 62f3e3d6-e8a2-4e76-9954-60425ae15a04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: 4278d34b-1456-4b6a-b349-dc07785eda2f kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:32Z" name: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg.18a6ea675a3eb6f2 namespace: kserve-ci-e2e-test resourceVersion: "24734" uid: 74d2f7d5-ae14-4e6b-b38b-473fe7e1b253 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e9680-predictor-8689cf6cdf namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: 5ecae413-28a0-4b73-b05c-c0618faff328 kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: 'Created pod: error-404-isvc-e9680-predictor-8689cf6cdf-k7trg' metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:08Z" name: error-404-isvc-e9680-predictor-8689cf6cdf.18a6e9e3fb693b46 namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: 1fd4de5b-109f-4896-b510-e21c063a8877 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: e176d6bd-a366-411b-9cbd-2893f1f52fd2 kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: Scaled up replica set error-404-isvc-e9680-predictor-8689cf6cdf from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:08Z" name: error-404-isvc-e9680-predictor.18a6e9e3f1390fef namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: 93e5e197-d6bf-48f9-acdf-24d2e1918dcb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 7ecdc719-c5db-4e40-aa7d-1e6b8ffa5a37 kind: Event lastTimestamp: "2026-04-16T18:31:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:23Z" name: error-404-isvc-e9680-predictor.18a6e9e770d3b9b8 namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 785b826c-0ffb-4371-8db0-dd3d14e2232b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 7ecdc719-c5db-4e40-aa7d-1e6b8ffa5a37 kind: Event lastTimestamp: "2026-04-16T18:31:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:23Z" name: error-404-isvc-e9680-predictor.18a6e9e7718c07d4 namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: 8fdb4015-83e7-4013-abce-2e7669ca0265 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:31:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20650" uid: 7ecdc719-c5db-4e40-aa7d-1e6b8ffa5a37 kind: Event lastTimestamp: "2026-04-16T18:32:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:23Z" name: error-404-isvc-e9680-predictor.18a6e9eaf04e7f55 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: c42cfa34-0ee7-43ea-9011-bc5d2f53acb5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:31:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20650" uid: 7ecdc719-c5db-4e40-aa7d-1e6b8ffa5a37 kind: Event lastTimestamp: "2026-04-16T18:32:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:23Z" name: error-404-isvc-e9680-predictor.18a6e9eaf0b4571d namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: c754dd9b-cdfa-4ec7-9ab1-faa34197b88c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e9680 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: ebe887b1-6ad3-45f5-a3dd-0f10b5a8ec26 kind: Event lastTimestamp: "2026-04-16T18:32:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:32:00Z" name: error-404-isvc-e9680.18a6e9e3f1b1a0b6 namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: a08d9df0-d482-42ac-8ae4-98113ce5f12b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e9680 namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: ebe887b1-6ad3-45f5-a3dd-0f10b5a8ec26 kind: Event lastTimestamp: "2026-04-16T18:32:00Z" message: InferenceService [error-404-isvc-e9680] is Ready metadata: creationTimestamp: "2026-04-16T18:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:32:00Z" name: error-404-isvc-e9680.18a6e9f00c28a848 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 0a677a1a-0eee-402c-8eb4-1761cad46b66 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:18:02.213477Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13432" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:18:02Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e92ced4f8bd8 namespace: kserve-ci-e2e-test resourceVersion: "13438" uid: f3805c49-ad0e-4e05-8db7-0c1f24ebfebf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13442" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:18:02Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e92d06cd5385 namespace: kserve-ci-e2e-test resourceVersion: "13539" uid: 73e93d3b-164c-40b9-92f9-4fa40c2e0710 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:44a3134685c61cdead52dd49e24582d49c2855142a8651c186f37c9fe776f06c" metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:02Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e92d084f0df4 namespace: kserve-ci-e2e-test resourceVersion: "13543" uid: e8f2a6a4-7335-4f41-85dd-919dd7ea2f8f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:18:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:44a3134685c61cdead52dd49e24582d49c2855142a8651c186f37c9fe776f06c" in 4.829s (4.829s including waiting). Image size: 300960178 bytes.' metadata: creationTimestamp: "2026-04-16T18:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:07Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e92e282646a0 namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: 68e50ace-2a2a-42ce-9572-1c6d47d4d753 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:18:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:07Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e92e2b13cba8 namespace: kserve-ci-e2e-test resourceVersion: "13614" uid: aabdf631-4122-43bb-9bce-21ffc13c566a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:18:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:07Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e92e2bcbe686 namespace: kserve-ci-e2e-test resourceVersion: "13617" uid: 2b5742fd-e642-4d0e-81b7-a2bbccf54860 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:18:11Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1374" metadata: creationTimestamp: "2026-04-16T18:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:11Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e92f19e29ee6 namespace: kserve-ci-e2e-test resourceVersion: "13664" uid: 46e9a05a-8803-423e-8c4a-da0cd6d2356d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:18:36Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1374" in 25.048s (25.048s including waiting). Image size: 1560610740 bytes.' metadata: creationTimestamp: "2026-04-16T18:18:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:36Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e934eeddf824 namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: 1f6f5f0f-1bcd-4f21-8020-bb2aa7bf3112 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:18:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:18:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:36Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e934f5173994 namespace: kserve-ci-e2e-test resourceVersion: "13906" uid: 62d8c6dc-d7cf-4685-8ca6-6ad68c5f6cfa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:18:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:18:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:36Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e934f6436a30 namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 57df3f4a-3546-4b4c-86ea-3b740be9f26e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:18:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:20:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:25Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e9352d7cc6bc namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 019d02ee-67e6-429a-ba87-1fdb24f65f0c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: cb6437f7-3993-4858-a222-e997b89c17a6 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr.18a6e94d952ca640 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: fd4670fe-cd4c-4b6d-b647-597d40841bff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-69d5d56664 namespace: kserve-ci-e2e-test resourceVersion: "13420" uid: 1fcc2eed-5b83-4651-a1b6-70a07cd299f5 kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-69d5d56664-9jmvr' metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: isvc-sklearn-graph-1-predictor-69d5d56664.18a6e92ceced59ba namespace: kserve-ci-e2e-test resourceVersion: "13435" uid: 90632350-d473-4e76-9373-d7dc83d12e5c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13418" uid: a2e0f4c9-8e1d-4f8c-963c-7a6c2b184e5c kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-69d5d56664 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: isvc-sklearn-graph-1-predictor.18a6e92ce6a7cbc9 namespace: kserve-ci-e2e-test resourceVersion: "13421" uid: c3e7ab45-8979-444f-aecd-6e3429852b48 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13431" uid: 42927e98-1c30-4976-b10c-7417680ec628 kind: Event lastTimestamp: "2026-04-16T18:18:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:47Z" name: isvc-sklearn-graph-1-predictor.18a6e9306bc53520 namespace: kserve-ci-e2e-test resourceVersion: "13991" uid: 3c38f80b-82b4-4284-a634-d59086611e17 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13431" uid: 42927e98-1c30-4976-b10c-7417680ec628 kind: Event lastTimestamp: "2026-04-16T18:18:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:47Z" name: isvc-sklearn-graph-1-predictor.18a6e9306c2ed49f namespace: kserve-ci-e2e-test resourceVersion: "13992" uid: 852e5832-c122-4974-b376-d290a94efb28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:19:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13708" uid: 42927e98-1c30-4976-b10c-7417680ec628 kind: Event lastTimestamp: "2026-04-16T18:20:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:17Z" name: isvc-sklearn-graph-1-predictor.18a6e93aea1d2735 namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: d3443b61-eec5-4d9f-8990-ec51c1db4729 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:19:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13708" uid: 42927e98-1c30-4976-b10c-7417680ec628 kind: Event lastTimestamp: "2026-04-16T18:20:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:17Z" name: isvc-sklearn-graph-1-predictor.18a6e93aeaa17f0a namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: c8fce8c1-7d4f-4ebd-afdd-678f3aed77e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13402" uid: 7cf0a20a-c2f3-4034-bbbd-6a197192c9da kind: Event lastTimestamp: "2026-04-16T18:19:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:46Z" name: isvc-sklearn-graph-1.18a6e92cecb02da5 namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: 125241c3-b5ab-4bc0-b0ae-dacec5cf0ade reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: 7cf0a20a-c2f3-4034-bbbd-6a197192c9da kind: Event lastTimestamp: "2026-04-16T18:19:46Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-16T18:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:46Z" name: isvc-sklearn-graph-1.18a6e9451add038f namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: b506dddd-3807-4f2b-a574-34bee781ee53 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:18:02.732360Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:18:02Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e92d0c3d139d namespace: kserve-ci-e2e-test resourceVersion: "13551" uid: bb8d8060-4d94-4a77-b5f0-eeea263e8a7e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13554" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:18:03Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:18:03Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e92d288ced08 namespace: kserve-ci-e2e-test resourceVersion: "13567" uid: e035cbc1-a952-4efc-95a6-c3c3f8672ea5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:18:03Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:44a3134685c61cdead52dd49e24582d49c2855142a8651c186f37c9fe776f06c" metadata: creationTimestamp: "2026-04-16T18:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:03Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e92d29f16e6d namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 84db2bee-7eeb-48a6-ba1a-d1427989bd77 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:18:19Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:44a3134685c61cdead52dd49e24582d49c2855142a8651c186f37c9fe776f06c" in 16.044s (16.044s including waiting). Image size: 300960178 bytes.' metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:19Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e930e6482a5c namespace: kserve-ci-e2e-test resourceVersion: "13736" uid: 43f2f6a3-970f-430d-ad65-6f91540dc0b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:18:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:19Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e930e9bb5b21 namespace: kserve-ci-e2e-test resourceVersion: "13739" uid: f6b6ad75-7d2a-4dae-a629-88b3b0eaa726 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:18:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:19Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e930ea212282 namespace: kserve-ci-e2e-test resourceVersion: "13740" uid: 9fa46bfd-ba66-43fe-9c62-06655997861e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:18:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1374" metadata: creationTimestamp: "2026-04-16T18:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:23Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e931e8417e88 namespace: kserve-ci-e2e-test resourceVersion: "13792" uid: 9387de8e-acaa-4b24-8868-af0e826cd4e6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:18:30Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1374" in 7.129s (7.129s including waiting). Image size: 1560610740 bytes.' metadata: creationTimestamp: "2026-04-16T18:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:30Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e933912f9b89 namespace: kserve-ci-e2e-test resourceVersion: "13852" uid: 8c99bd52-013d-4535-bef9-6520a5d5640a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:18:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:30Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e9339462df49 namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: 720936a4-efb8-4eb1-b7da-25e346681068 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:18:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:30Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e93394d1651f namespace: kserve-ci-e2e-test resourceVersion: "13855" uid: 64439efd-0f8f-4c75-83d5-3b254ea2b2c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T18:18:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:18:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e933c6fef5be namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 0188e2d6-bc67-4dd9-8e46-3b1791d78a8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 41d6b77e-7da9-4457-916e-62779d3f7094 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn.18a6e94d9146da56 namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: 9c05b590-04c3-436c-9e86-0809579cd904 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7c54695cb6 namespace: kserve-ci-e2e-test resourceVersion: "13523" uid: e038dc2c-1f74-40c7-85f2-9749425fa80c kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7c54695cb6-zhrvn' metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6.18a6e92d0bd50253 namespace: kserve-ci-e2e-test resourceVersion: "13548" uid: a2274985-30a0-45f1-a5c8-aacb77f4409e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13519" uid: 3fc129ae-aebf-4012-b667-6b11aa1cfbdd kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7c54695cb6 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: isvc-sklearn-graph-2-predictor.18a6e92cff89f91f namespace: kserve-ci-e2e-test resourceVersion: "13526" uid: 81759ba9-ed8e-47fb-bbc9-75ec87ce5ab2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13522" uid: 76318eb9-f335-4e75-a024-5aeeead260c0 kind: Event lastTimestamp: "2026-04-16T18:19:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:02Z" name: isvc-sklearn-graph-2-predictor.18a6e9307e3fc6f7 namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 6802289c-e05d-4711-bb4b-1d65d8e0048d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13522" uid: 76318eb9-f335-4e75-a024-5aeeead260c0 kind: Event lastTimestamp: "2026-04-16T18:19:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:02Z" name: isvc-sklearn-graph-2-predictor.18a6e9307eac5a35 namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: 23197d6d-7d89-4cc8-a192-0eb1de8514ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:19:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: 76318eb9-f335-4e75-a024-5aeeead260c0 kind: Event lastTimestamp: "2026-04-16T18:20:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:02Z" name: isvc-sklearn-graph-2-predictor.18a6e93e7a3de74e namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: 0dc21431-ce44-4d9c-9073-864e04b5b34e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:19:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: 76318eb9-f335-4e75-a024-5aeeead260c0 kind: Event lastTimestamp: "2026-04-16T18:20:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:02Z" name: isvc-sklearn-graph-2-predictor.18a6e93e7aa8f3e3 namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: d68cf9fe-5913-4131-8c5e-bffad9a02921 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13509" uid: 1d9c5a79-226d-4bd7-af07-49e90aa994f0 kind: Event lastTimestamp: "2026-04-16T18:19:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:41Z" name: isvc-sklearn-graph-2.18a6e92cff8cfb88 namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: a024cbbb-5731-4eaa-964a-2b20accb5610 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 1d9c5a79-226d-4bd7-af07-49e90aa994f0 kind: Event lastTimestamp: "2026-04-16T18:19:41Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-16T18:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:41Z" name: isvc-sklearn-graph-2.18a6e94426eed320 namespace: kserve-ci-e2e-test resourceVersion: "14637" uid: fe74c048-ee18-4de6-aa0a-5af1959815a3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:18:02.413466Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13484" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-89nm4 to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:18:02Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e92cf93b1c9e namespace: kserve-ci-e2e-test resourceVersion: "13490" uid: a30de96c-7ab3-4131-910d-c932514344b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13498" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:18:02Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e92d12b5d798 namespace: kserve-ci-e2e-test resourceVersion: "13558" uid: 6644eb40-1813-4465-9634-ca27ebbf7935 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:44a3134685c61cdead52dd49e24582d49c2855142a8651c186f37c9fe776f06c" metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:02Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e92d13f4ed6a namespace: kserve-ci-e2e-test resourceVersion: "13560" uid: e7b80c5a-1d20-439e-afea-82b913273fb1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:18:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:44a3134685c61cdead52dd49e24582d49c2855142a8651c186f37c9fe776f06c" in 4.623s (4.623s including waiting). Image size: 300960178 bytes.' metadata: creationTimestamp: "2026-04-16T18:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:07Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e92e27876774 namespace: kserve-ci-e2e-test resourceVersion: "13610" uid: c58cf9d4-6516-4273-b1fe-384693a7a250 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:18:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:07Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e92e2af26fbd namespace: kserve-ci-e2e-test resourceVersion: "13613" uid: a01283e4-7221-492f-84fb-2865a9bf0e09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:18:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:07Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e92e2ba825ab namespace: kserve-ci-e2e-test resourceVersion: "13615" uid: 7072315b-c8f9-47b8-a50c-5c5482a61665 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:18:11Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T18:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:11Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e92f19d3ecb5 namespace: kserve-ci-e2e-test resourceVersion: "13663" uid: a03a6913-aae7-42af-bbbb-5b4bda0239eb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:18:37Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 25.526s (25.526s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T18:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:37Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e9350b5093f8 namespace: kserve-ci-e2e-test resourceVersion: "13912" uid: f22f63a3-6458-4314-9f1b-8c0b069fc862 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:18:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:37Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e93511bfd802 namespace: kserve-ci-e2e-test resourceVersion: "13913" uid: 2a561e9e-b4f7-4464-9f21-fd1095ff1a09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:18:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:37Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e93512f437c4 namespace: kserve-ci-e2e-test resourceVersion: "13914" uid: b099a2f3-f3ea-4b8a-aa14-6a7628293510 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:18:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:19:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:28Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e9352d91118f namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: 0de79249-ceab-4cca-8908-60f92a870f66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-89nm4 namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 5bba13cd-dc2d-45fe-ba30-3cdccac702a1 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: isvc-xgboost-graph-predictor-784cb989b8-89nm4.18a6e94d8e57c466 namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: 1a59da42-5b0a-4dea-a28d-56a92f069ca9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "13476" uid: d196f8ac-1e42-4e2b-b36b-c090b4f10119 kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-89nm4' metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a6e92cf8cd2813 namespace: kserve-ci-e2e-test resourceVersion: "13486" uid: bf70eb1d-05db-426f-93fa-07a658b79382 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13474" uid: 530167bd-bc6f-4462-ba27-c0e87bfbd26e kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: isvc-xgboost-graph-predictor.18a6e92cf602174e namespace: kserve-ci-e2e-test resourceVersion: "13478" uid: e5fdf7d2-0557-4b01-a4fd-7b9b79c24fec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13475" uid: 88a4e63e-bce0-440c-941e-100b576cb893 kind: Event lastTimestamp: "2026-04-16T18:19:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:17Z" name: isvc-xgboost-graph-predictor.18a6e930748e11d2 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: f8ee1f7a-6145-47a7-8d8a-2f4ef3bbdd1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13475" uid: 88a4e63e-bce0-440c-941e-100b576cb893 kind: Event lastTimestamp: "2026-04-16T18:19:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:17Z" name: isvc-xgboost-graph-predictor.18a6e93074f55827 namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: 7cbacdd1-c661-4af1-bb70-f890c8a22b55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13714" uid: 88a4e63e-bce0-440c-941e-100b576cb893 kind: Event lastTimestamp: "2026-04-16T18:20:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:02Z" name: isvc-xgboost-graph-predictor.18a6e941ef8aac0f namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: fef398d0-17f8-4bfa-a1ce-2870c4037484 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13714" uid: 88a4e63e-bce0-440c-941e-100b576cb893 kind: Event lastTimestamp: "2026-04-16T18:20:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:02Z" name: isvc-xgboost-graph-predictor.18a6e941f003e2e6 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: ff4aac21-2ccf-4348-be39-e7c5e3dc2f98 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13463" uid: 99d671e7-f570-4c4f-86a5-ccabf40efe69 kind: Event lastTimestamp: "2026-04-16T18:19:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:38Z" name: isvc-xgboost-graph.18a6e92cf5d831e3 namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 43e125ca-c0ae-4b6f-8fdb-81022597b01f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: 99d671e7-f570-4c4f-86a5-ccabf40efe69 kind: Event lastTimestamp: "2026-04-16T18:19:38Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-16T18:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:38Z" name: isvc-xgboost-graph.18a6e943658109a2 namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: f2094127-b4fb-4d8d-845e-0fdabeb21d0e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:20:12.170719Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-55559dfcf8-vvdns namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: a2841e7a-1489-4ee2-867d-68ad0e8e8a68 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-55559dfcf8-vvdns to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:20:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:20:12Z" name: model-chainer-55559dfcf8-vvdns.18a6e94b2f5dac92 namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: e5ca4f84-2149-4249-b08e-3d3f7122a2f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:12Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-55559dfcf8-vvdns namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: a2841e7a-1489-4ee2-867d-68ad0e8e8a68 kind: Event lastTimestamp: "2026-04-16T18:20:12Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:20:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:20:12Z" name: model-chainer-55559dfcf8-vvdns.18a6e94b48ac404d namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 2522d413-2451-4ecd-ae66-82d779bfb0ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55559dfcf8-vvdns namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: a2841e7a-1489-4ee2-867d-68ad0e8e8a68 kind: Event lastTimestamp: "2026-04-16T18:20:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:20:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:12Z" name: model-chainer-55559dfcf8-vvdns.18a6e94b4b18327d namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 9760719b-c5ea-45e3-bace-673174f3e637 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55559dfcf8-vvdns namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: a2841e7a-1489-4ee2-867d-68ad0e8e8a68 kind: Event lastTimestamp: "2026-04-16T18:20:12Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-16T18:20:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:12Z" name: model-chainer-55559dfcf8-vvdns.18a6e94b4dff76dd namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 5b6f6565-7715-427e-9c7b-b719b618125e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55559dfcf8-vvdns namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: a2841e7a-1489-4ee2-867d-68ad0e8e8a68 kind: Event lastTimestamp: "2026-04-16T18:20:12Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-16T18:20:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:12Z" name: model-chainer-55559dfcf8-vvdns.18a6e94b4e6a29d7 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: 328f6166-6166-44d9-87e6-54a71021200c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55559dfcf8-vvdns namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: a2841e7a-1489-4ee2-867d-68ad0e8e8a68 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: model-chainer-55559dfcf8-vvdns.18a6e94d8aa25f27 namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: e3b2a67c-efab-4f80-b801-104f50bca146 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:20:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55559dfcf8-vvdns namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: a2841e7a-1489-4ee2-867d-68ad0e8e8a68 kind: Event lastTimestamp: "2026-04-16T18:20:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:20:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:48Z" name: model-chainer-55559dfcf8-vvdns.18a6e94ded2756cc namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: a8526a53-4ecd-4739-a0e1-e6da94e84be4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-55559dfcf8 namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: 6121eac7-8a4e-4b44-a93a-21b99eeec4c1 kind: Event lastTimestamp: "2026-04-16T18:20:12Z" message: 'Created pod: model-chainer-55559dfcf8-vvdns' metadata: creationTimestamp: "2026-04-16T18:20:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:12Z" name: model-chainer-55559dfcf8.18a6e94b2ef8d4b4 namespace: kserve-ci-e2e-test resourceVersion: "14905" uid: d14bff60-bb60-4e86-a1be-719f170e12f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: 8dacdc94-b162-4e0f-b4ae-6a58be67262e kind: Event lastTimestamp: "2026-04-16T18:20:12Z" message: Scaled up replica set model-chainer-55559dfcf8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:20:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:12Z" name: model-chainer.18a6e94b2e0e04fa namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: fd97fc86-6e5a-4fb6-a249-6becee7fb0d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 640e62a3-f0ed-490e-bb50-b3b7db626a4a kind: Event lastTimestamp: "2026-04-16T18:20:12Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-16T18:20:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:20:12Z" name: model-chainer.18a6e94b31c5698e namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 78af34a4-99b3-49cd-b448-3a6d084aaea5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:21:32.486926Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-68d6a-5bfbb5d456-zkhmz namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: cb05b91f-71b6-4c5c-ba47-b4c77cfa716d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-68d6a-5bfbb5d456-zkhmz to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:21:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:21:32Z" name: sequence-graph-68d6a-5bfbb5d456-zkhmz.18a6e95de295b279 namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: f26f9b2e-c7cf-4e83-a696-4528cb538fa0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:32Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-68d6a-5bfbb5d456-zkhmz namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: cb05b91f-71b6-4c5c-ba47-b4c77cfa716d kind: Event lastTimestamp: "2026-04-16T18:21:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-68d6a-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:21:32Z" name: sequence-graph-68d6a-5bfbb5d456-zkhmz.18a6e95dedab5ccd namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 00814f26-6316-4156-aa5d-eeef86518143 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:33Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-68d6a-5bfbb5d456-zkhmz namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: cb05b91f-71b6-4c5c-ba47-b4c77cfa716d kind: Event lastTimestamp: "2026-04-16T18:21:33Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:21:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:21:33Z" name: sequence-graph-68d6a-5bfbb5d456-zkhmz.18a6e95e1f97e31f namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 4a7e1563-ebc3-4b1f-84e3-8c0bc868c18e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-68d6a} kind: Pod name: sequence-graph-68d6a-5bfbb5d456-zkhmz namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: cb05b91f-71b6-4c5c-ba47-b4c77cfa716d kind: Event lastTimestamp: "2026-04-16T18:21:33Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:21:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:21:33Z" name: sequence-graph-68d6a-5bfbb5d456-zkhmz.18a6e95e20cd8d74 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 9dce8ffd-f993-4a58-ba4e-91fa37929d2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-68d6a} kind: Pod name: sequence-graph-68d6a-5bfbb5d456-zkhmz namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: cb05b91f-71b6-4c5c-ba47-b4c77cfa716d kind: Event lastTimestamp: "2026-04-16T18:21:33Z" message: 'Created container: sequence-graph-68d6a' metadata: creationTimestamp: "2026-04-16T18:21:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:21:33Z" name: sequence-graph-68d6a-5bfbb5d456-zkhmz.18a6e95e23840894 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: b9e78b1e-9632-4795-be0a-5d2eefb14d1f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-68d6a} kind: Pod name: sequence-graph-68d6a-5bfbb5d456-zkhmz namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: cb05b91f-71b6-4c5c-ba47-b4c77cfa716d kind: Event lastTimestamp: "2026-04-16T18:21:33Z" message: Started container sequence-graph-68d6a metadata: creationTimestamp: "2026-04-16T18:21:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:21:33Z" name: sequence-graph-68d6a-5bfbb5d456-zkhmz.18a6e95e23ff465b namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: aeb624df-82d0-4aaa-9e5f-36862dc946e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-68d6a} kind: Pod name: sequence-graph-68d6a-5bfbb5d456-zkhmz namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: cb05b91f-71b6-4c5c-ba47-b4c77cfa716d kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Stopping container sequence-graph-68d6a metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:47Z" name: sequence-graph-68d6a-5bfbb5d456-zkhmz.18a6e9d118b704a4 namespace: kserve-ci-e2e-test resourceVersion: "19378" uid: 8e9e5cc1-ea47-406c-a5a2-f50673b3b4bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:29:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-68d6a} kind: Pod name: sequence-graph-68d6a-5bfbb5d456-zkhmz namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: cb05b91f-71b6-4c5c-ba47-b4c77cfa716d kind: Event lastTimestamp: "2026-04-16T18:30:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:29:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:15Z" name: sequence-graph-68d6a-5bfbb5d456-zkhmz.18a6e9d1c46f62e0 namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: abec2646-1aac-4cf7-b74e-46e91e6020a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-68d6a-5bfbb5d456 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 5d3e1607-55db-4743-9f94-a555cbf7f460 kind: Event lastTimestamp: "2026-04-16T18:21:32Z" message: 'Created pod: sequence-graph-68d6a-5bfbb5d456-zkhmz' metadata: creationTimestamp: "2026-04-16T18:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:32Z" name: sequence-graph-68d6a-5bfbb5d456.18a6e95de1efc056 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: f334ef65-5e4d-488a-8829-2455a26b84fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: e7988df6-0542-4e60-9753-8d8e4a6a2a67 kind: Event lastTimestamp: "2026-04-16T18:21:32Z" message: Scaled up replica set sequence-graph-68d6a-5bfbb5d456 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:32Z" name: sequence-graph-68d6a.18a6e95de0fe1ddd namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: a13ae803-448e-4254-8a7e-77cfa440e1a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 2e68109e-e4ed-4cd2-81af-0eee5f532b46 kind: Event lastTimestamp: "2026-04-16T18:21:32Z" message: InferenceGraph [sequence-graph-68d6a] is Ready metadata: creationTimestamp: "2026-04-16T18:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:21:32Z" name: sequence-graph-68d6a.18a6e95de4ec6a3e namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: d22316a0-0b20-43c7-b927-321c46387c26 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 7f1dd5e4-279b-4451-bbaa-f4f4dfe49380 kind: Event lastTimestamp: "2026-04-16T18:21:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:47Z" name: sequence-graph-68d6a.18a6e9615f855043 namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 159b62cf-0d7a-4eb4-ad75-ddb1dd8e7987 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 7f1dd5e4-279b-4451-bbaa-f4f4dfe49380 kind: Event lastTimestamp: "2026-04-16T18:21:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:47Z" name: sequence-graph-68d6a.18a6e9615ff8a539 namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: b03a2250-3102-4a6b-8e74-193d0fa1bb3e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:22:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 7f1dd5e4-279b-4451-bbaa-f4f4dfe49380 kind: Event lastTimestamp: "2026-04-16T18:22:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:22:02Z" name: sequence-graph-68d6a.18a6e964debb9fd6 namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: ab2b8839-067e-47a8-89c6-04a1c50c4fc9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:22:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 7f1dd5e4-279b-4451-bbaa-f4f4dfe49380 kind: Event lastTimestamp: "2026-04-16T18:22:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:22:02Z" name: sequence-graph-68d6a.18a6e964df260637 namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 82c8f065-5a35-4b84-a593-634efb926e2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:30:57.638563Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c659a-cd9d4c6d8-d4jbv namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: 2d259fc2-8650-4916-b45f-ade258b42eb4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c659a-cd9d4c6d8-d4jbv to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:30:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:30:57Z" name: sequence-graph-c659a-cd9d4c6d8-d4jbv.18a6e9e1783f5bd4 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: 367fb7eb-895f-4fd6-99d9-8de5b6b63692 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:58Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c659a-cd9d4c6d8-d4jbv namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: 2d259fc2-8650-4916-b45f-ade258b42eb4 kind: Event lastTimestamp: "2026-04-16T18:30:58Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:30:58Z" name: sequence-graph-c659a-cd9d4c6d8-d4jbv.18a6e9e191636d94 namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 83457286-b738-4d8f-bacf-a91aefae1b1d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c659a} kind: Pod name: sequence-graph-c659a-cd9d4c6d8-d4jbv namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 2d259fc2-8650-4916-b45f-ade258b42eb4 kind: Event lastTimestamp: "2026-04-16T18:30:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:58Z" name: sequence-graph-c659a-cd9d4c6d8-d4jbv.18a6e9e193870fba namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: 84757f86-4519-4e9c-9d73-00db20193cc6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c659a} kind: Pod name: sequence-graph-c659a-cd9d4c6d8-d4jbv namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 2d259fc2-8650-4916-b45f-ade258b42eb4 kind: Event lastTimestamp: "2026-04-16T18:30:58Z" message: 'Created container: sequence-graph-c659a' metadata: creationTimestamp: "2026-04-16T18:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:58Z" name: sequence-graph-c659a-cd9d4c6d8-d4jbv.18a6e9e1966da2dd namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: 94283793-a131-4145-a15c-89d37595bea4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c659a} kind: Pod name: sequence-graph-c659a-cd9d4c6d8-d4jbv namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 2d259fc2-8650-4916-b45f-ade258b42eb4 kind: Event lastTimestamp: "2026-04-16T18:30:58Z" message: Started container sequence-graph-c659a metadata: creationTimestamp: "2026-04-16T18:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:58Z" name: sequence-graph-c659a-cd9d4c6d8-d4jbv.18a6e9e1972a4504 namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: 7752541b-bf88-4567-bf6a-451d9b9e0889 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c659a} kind: Pod name: sequence-graph-c659a-cd9d4c6d8-d4jbv namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 2d259fc2-8650-4916-b45f-ade258b42eb4 kind: Event lastTimestamp: "2026-04-16T18:31:07Z" message: Stopping container sequence-graph-c659a metadata: creationTimestamp: "2026-04-16T18:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:07Z" name: sequence-graph-c659a-cd9d4c6d8-d4jbv.18a6e9e3d15805d5 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: a186ba81-38a6-4ba0-8d38-88c56192a5c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c659a} kind: Pod name: sequence-graph-c659a-cd9d4c6d8-d4jbv namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 2d259fc2-8650-4916-b45f-ade258b42eb4 kind: Event lastTimestamp: "2026-04-16T18:31:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:34Z" name: sequence-graph-c659a-cd9d4c6d8-d4jbv.18a6e9e44fe5ea3c namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: e56ed1c8-4618-4f72-ac42-844baf8e94d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c659a-cd9d4c6d8 namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: 08561b6d-2f29-4346-ba01-e9beeb5f3190 kind: Event lastTimestamp: "2026-04-16T18:30:57Z" message: 'Created pod: sequence-graph-c659a-cd9d4c6d8-d4jbv' metadata: creationTimestamp: "2026-04-16T18:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:57Z" name: sequence-graph-c659a-cd9d4c6d8.18a6e9e1770180d1 namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: a0569458-0cf0-4347-8b6b-9ab633f4af1d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c659a namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 5371deea-e01f-44d0-88c2-20a4ada67542 kind: Event lastTimestamp: "2026-04-16T18:30:57Z" message: Scaled up replica set sequence-graph-c659a-cd9d4c6d8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:57Z" name: sequence-graph-c659a.18a6e9e174a6bbdb namespace: kserve-ci-e2e-test resourceVersion: "20272" uid: 09a284ef-41ed-4401-b598-11d125896305 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c659a namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: aeeafd97-032a-4275-9f17-a41a285f4194 kind: Event lastTimestamp: "2026-04-16T18:30:57Z" message: InferenceGraph [sequence-graph-c659a] is Ready metadata: creationTimestamp: "2026-04-16T18:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:57Z" name: sequence-graph-c659a.18a6e9e181fba1ed namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: e32d246a-de6b-45da-af21-faeaf485cbc7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c659a namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: aeeafd97-032a-4275-9f17-a41a285f4194 kind: Event lastTimestamp: "2026-04-16T18:30:57Z" message: 'Failed to update status for InferenceGraph "sequence-graph-c659a": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-c659a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:57Z" name: sequence-graph-c659a.18a6e9e187d758e0 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: ef6fb53e-52e9-4d69-86f6-18244473c46b reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c659a namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: aeeafd97-032a-4275-9f17-a41a285f4194 kind: Event lastTimestamp: "2026-04-16T18:30:57Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-c659a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:57Z" name: sequence-graph-c659a.18a6e9e187d7aead namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: ff6d7e12-7fe9-4bdb-8f4a-72a1dc84da61 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:32:18.016791Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e9680-6cdfbbbc6f-c6qcf namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: b6db6f19-b698-45c4-8a3e-77fe8b193239 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e9680-6cdfbbbc6f-c6qcf to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:32:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:32:18Z" name: sequence-graph-e9680-6cdfbbbc6f-c6qcf.18a6e9f42f29bdd1 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 4226da5a-fd47-4fc2-b6cb-a01f8351de8b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:18Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e9680-6cdfbbbc6f-c6qcf namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: b6db6f19-b698-45c4-8a3e-77fe8b193239 kind: Event lastTimestamp: "2026-04-16T18:32:18Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:32:18Z" name: sequence-graph-e9680-6cdfbbbc6f-c6qcf.18a6e9f448304d93 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 599fccb1-c745-4918-9830-6e9719bf1709 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e9680} kind: Pod name: sequence-graph-e9680-6cdfbbbc6f-c6qcf namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: b6db6f19-b698-45c4-8a3e-77fe8b193239 kind: Event lastTimestamp: "2026-04-16T18:32:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:32:18Z" name: sequence-graph-e9680-6cdfbbbc6f-c6qcf.18a6e9f4498af155 namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: 142775c4-748b-44c3-8b92-a4d9cee3528e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e9680} kind: Pod name: sequence-graph-e9680-6cdfbbbc6f-c6qcf namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: b6db6f19-b698-45c4-8a3e-77fe8b193239 kind: Event lastTimestamp: "2026-04-16T18:32:18Z" message: 'Created container: sequence-graph-e9680' metadata: creationTimestamp: "2026-04-16T18:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:32:18Z" name: sequence-graph-e9680-6cdfbbbc6f-c6qcf.18a6e9f44c8e2169 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 5a0f89fa-93e9-45bb-aae2-13179cebb2df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e9680} kind: Pod name: sequence-graph-e9680-6cdfbbbc6f-c6qcf namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: b6db6f19-b698-45c4-8a3e-77fe8b193239 kind: Event lastTimestamp: "2026-04-16T18:32:18Z" message: Started container sequence-graph-e9680 metadata: creationTimestamp: "2026-04-16T18:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:32:18Z" name: sequence-graph-e9680-6cdfbbbc6f-c6qcf.18a6e9f44cfd9d1f namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: dffc3b92-1735-4b70-ba3e-f5a1e0bb60a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e9680} kind: Pod name: sequence-graph-e9680-6cdfbbbc6f-c6qcf namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: b6db6f19-b698-45c4-8a3e-77fe8b193239 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Stopping container sequence-graph-e9680 metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:32Z" name: sequence-graph-e9680-6cdfbbbc6f-c6qcf.18a6ea6755c8d4db namespace: kserve-ci-e2e-test resourceVersion: "24716" uid: 0c4ab354-be87-434d-9e0c-0e695ad41edc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e9680} kind: Pod name: sequence-graph-e9680-6cdfbbbc6f-c6qcf namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: b6db6f19-b698-45c4-8a3e-77fe8b193239 kind: Event lastTimestamp: "2026-04-16T18:41:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:00Z" name: sequence-graph-e9680-6cdfbbbc6f-c6qcf.18a6ea67eaf6d825 namespace: kserve-ci-e2e-test resourceVersion: "25103" uid: 48a1ea81-db98-488e-9451-0cd3084ddfe4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e9680-6cdfbbbc6f namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 2e481038-d9df-4142-a8a7-c212a395c9ae kind: Event lastTimestamp: "2026-04-16T18:32:18Z" message: 'Created pod: sequence-graph-e9680-6cdfbbbc6f-c6qcf' metadata: creationTimestamp: "2026-04-16T18:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:18Z" name: sequence-graph-e9680-6cdfbbbc6f.18a6e9f42ec2dba8 namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 3676e370-674a-45f1-a823-6b40ce20fc87 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e9680 namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: 7fe6f856-cf1e-454a-ba6a-960c17819ef4 kind: Event lastTimestamp: "2026-04-16T18:32:17Z" message: Scaled up replica set sequence-graph-e9680-6cdfbbbc6f from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:17Z" name: sequence-graph-e9680.18a6e9f42de9887e namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: cd82637c-9164-4971-a396-d9ce89ac92b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e9680 namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: 21b89839-1c79-4a17-8863-c81f5cccad35 kind: Event lastTimestamp: "2026-04-16T18:32:18Z" message: InferenceGraph [sequence-graph-e9680] is Ready metadata: creationTimestamp: "2026-04-16T18:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:32:18Z" name: sequence-graph-e9680.18a6e9f4341f6565 namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 5795b8dd-5056-4c98-913f-06f7cd9d1367 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e9680 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 45becb7f-f577-41cd-8f4a-c93ad2813f10 kind: Event lastTimestamp: "2026-04-16T18:32:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:33Z" name: sequence-graph-e9680.18a6e9f7ac9d6c1a namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: 4f12814b-6507-49b4-8633-bda8df3377a2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e9680 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 45becb7f-f577-41cd-8f4a-c93ad2813f10 kind: Event lastTimestamp: "2026-04-16T18:32:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:33Z" name: sequence-graph-e9680.18a6e9f7acf62a37 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 1d529bbd-ac99-40c4-9700-c2cf7de4d558 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e9680 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 45becb7f-f577-41cd-8f4a-c93ad2813f10 kind: Event lastTimestamp: "2026-04-16T18:32:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:48Z" name: sequence-graph-e9680.18a6e9fb2bb245f0 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 18038d4c-f107-4ea3-acc7-59a6d3a209ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e9680 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 45becb7f-f577-41cd-8f4a-c93ad2813f10 kind: Event lastTimestamp: "2026-04-16T18:32:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:48Z" name: sequence-graph-e9680.18a6e9fb2c21b599 namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: d5fe7eca-1c15-4c74-862f-b8ecae6a0ad9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:42:13.323482Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1ac5a-67bcd68668-mm8sj namespace: kserve-ci-e2e-test resourceVersion: "25901" uid: 9f3f7f20-6cfe-4131-ad6a-809a0361a7b6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-1ac5a-67bcd68668-mm8sj to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:42:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:42:13Z" name: splitter-graph-1ac5a-67bcd68668-mm8sj.18a6ea7eca350419 namespace: kserve-ci-e2e-test resourceVersion: "25906" uid: 8c736708-e5a1-490e-8d98-ed200bac08b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:13Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1ac5a-67bcd68668-mm8sj namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 9f3f7f20-6cfe-4131-ad6a-809a0361a7b6 kind: Event lastTimestamp: "2026-04-16T18:42:13Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:42:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:42:13Z" name: splitter-graph-1ac5a-67bcd68668-mm8sj.18a6ea7ee33e85dd namespace: kserve-ci-e2e-test resourceVersion: "25919" uid: 34215038-4e71-45dd-8282-98aed7db4c38 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1ac5a} kind: Pod name: splitter-graph-1ac5a-67bcd68668-mm8sj namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: 9f3f7f20-6cfe-4131-ad6a-809a0361a7b6 kind: Event lastTimestamp: "2026-04-16T18:42:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:42:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:13Z" name: splitter-graph-1ac5a-67bcd68668-mm8sj.18a6ea7ee494abab namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: 0ece6ad1-5510-4b6e-9feb-3edf30a7d4fa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1ac5a} kind: Pod name: splitter-graph-1ac5a-67bcd68668-mm8sj namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: 9f3f7f20-6cfe-4131-ad6a-809a0361a7b6 kind: Event lastTimestamp: "2026-04-16T18:42:13Z" message: 'Created container: splitter-graph-1ac5a' metadata: creationTimestamp: "2026-04-16T18:42:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:13Z" name: splitter-graph-1ac5a-67bcd68668-mm8sj.18a6ea7ee792efa6 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: e8837186-c15f-4fbb-a9e8-6efbddc66427 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1ac5a} kind: Pod name: splitter-graph-1ac5a-67bcd68668-mm8sj namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: 9f3f7f20-6cfe-4131-ad6a-809a0361a7b6 kind: Event lastTimestamp: "2026-04-16T18:42:13Z" message: Started container splitter-graph-1ac5a metadata: creationTimestamp: "2026-04-16T18:42:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:42:13Z" name: splitter-graph-1ac5a-67bcd68668-mm8sj.18a6ea7ee80b2f5e namespace: kserve-ci-e2e-test resourceVersion: "25924" uid: 8b0d7b25-5a1e-4cc6-bec4-d7a51ad41c41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1ac5a} kind: Pod name: splitter-graph-1ac5a-67bcd68668-mm8sj namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: 9f3f7f20-6cfe-4131-ad6a-809a0361a7b6 kind: Event lastTimestamp: "2026-04-16T18:50:28Z" message: Stopping container splitter-graph-1ac5a metadata: creationTimestamp: "2026-04-16T18:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:50:28Z" name: splitter-graph-1ac5a-67bcd68668-mm8sj.18a6eaf1fbdff9bd namespace: kserve-ci-e2e-test resourceVersion: "29644" uid: 9057f8be-5d63-4c64-9a75-2a7cc7f53b5f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:50:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1ac5a} kind: Pod name: splitter-graph-1ac5a-67bcd68668-mm8sj namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: 9f3f7f20-6cfe-4131-ad6a-809a0361a7b6 kind: Event lastTimestamp: "2026-04-16T18:50:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:50:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:50:55Z" name: splitter-graph-1ac5a-67bcd68668-mm8sj.18a6eaf29f91c21f namespace: kserve-ci-e2e-test resourceVersion: "29874" uid: 6357fef3-b82e-41ea-8222-9758173952f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-1ac5a-67bcd68668 namespace: kserve-ci-e2e-test resourceVersion: "25898" uid: 2402a554-0990-43cb-b2c4-80dba530a0f1 kind: Event lastTimestamp: "2026-04-16T18:42:13Z" message: 'Created pod: splitter-graph-1ac5a-67bcd68668-mm8sj' metadata: creationTimestamp: "2026-04-16T18:42:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:13Z" name: splitter-graph-1ac5a-67bcd68668.18a6ea7ec9bd7e7d namespace: kserve-ci-e2e-test resourceVersion: "25902" uid: 89866154-5430-4c33-bfd5-cd45cf1b10e3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25896" uid: c0832aa7-b5b5-4c30-adc3-18e0f2980888 kind: Event lastTimestamp: "2026-04-16T18:42:13Z" message: Scaled up replica set splitter-graph-1ac5a-67bcd68668 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:42:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:13Z" name: splitter-graph-1ac5a.18a6ea7ec8ceede3 namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 291aa6c0-0a99-4f00-9f50-71aebabd6ca1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:42:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: fb4f0377-b945-4355-891d-2167e0a8e2ef kind: Event lastTimestamp: "2026-04-16T18:42:13Z" message: InferenceGraph [splitter-graph-1ac5a] is Ready metadata: creationTimestamp: "2026-04-16T18:42:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:42:13Z" name: splitter-graph-1ac5a.18a6ea7ecd3d1d90 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: 0f10a664-ad38-4303-b42a-d18ce068c85d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:42:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25897" uid: 6edb7e54-ad32-4478-95ab-fa18595a5684 kind: Event lastTimestamp: "2026-04-16T18:42:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:43Z" name: splitter-graph-1ac5a.18a6ea82475a1aa1 namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: 0fa717d0-5724-4bbb-a369-1498a7c000ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:42:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25897" uid: 6edb7e54-ad32-4478-95ab-fa18595a5684 kind: Event lastTimestamp: "2026-04-16T18:42:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:43Z" name: splitter-graph-1ac5a.18a6ea82479ffbe2 namespace: kserve-ci-e2e-test resourceVersion: "26307" uid: 7e12ef29-85f1-4593-83bb-9587dcd35b86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:40:52.991453Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e04f1-7c7f988c75-9hw7m namespace: kserve-ci-e2e-test resourceVersion: "25020" uid: a3fdb99e-7095-49cd-97d7-f6c00152f7d2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-e04f1-7c7f988c75-9hw7m to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:40:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:40:52Z" name: splitter-graph-e04f1-7c7f988c75-9hw7m.18a6ea6c160b8780 namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: 23dbb2dc-346b-4083-89dd-1b872239ef88 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:53Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e04f1-7c7f988c75-9hw7m namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: a3fdb99e-7095-49cd-97d7-f6c00152f7d2 kind: Event lastTimestamp: "2026-04-16T18:40:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-e04f1-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:53Z" name: splitter-graph-e04f1-7c7f988c75-9hw7m.18a6ea6c214e5e53 namespace: kserve-ci-e2e-test resourceVersion: "25038" uid: 88f5c4a5-fd75-4dfc-b029-2c58078e100f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:54Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e04f1-7c7f988c75-9hw7m namespace: kserve-ci-e2e-test resourceVersion: "25029" uid: a3fdb99e-7095-49cd-97d7-f6c00152f7d2 kind: Event lastTimestamp: "2026-04-16T18:40:54Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:40:54Z" name: splitter-graph-e04f1-7c7f988c75-9hw7m.18a6ea6c53130f99 namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: 1fc48818-b79f-44ae-9560-6f51cda25d86 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e04f1} kind: Pod name: splitter-graph-e04f1-7c7f988c75-9hw7m namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: a3fdb99e-7095-49cd-97d7-f6c00152f7d2 kind: Event lastTimestamp: "2026-04-16T18:40:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:54Z" name: splitter-graph-e04f1-7c7f988c75-9hw7m.18a6ea6c5493907f namespace: kserve-ci-e2e-test resourceVersion: "25048" uid: 325c2057-209a-4933-a86e-25924ad2a70e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e04f1} kind: Pod name: splitter-graph-e04f1-7c7f988c75-9hw7m namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: a3fdb99e-7095-49cd-97d7-f6c00152f7d2 kind: Event lastTimestamp: "2026-04-16T18:40:54Z" message: 'Created container: splitter-graph-e04f1' metadata: creationTimestamp: "2026-04-16T18:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:54Z" name: splitter-graph-e04f1-7c7f988c75-9hw7m.18a6ea6c57bcd700 namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: ac5de0ed-d3c5-4220-bf1b-ae4712c969fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e04f1} kind: Pod name: splitter-graph-e04f1-7c7f988c75-9hw7m namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: a3fdb99e-7095-49cd-97d7-f6c00152f7d2 kind: Event lastTimestamp: "2026-04-16T18:40:54Z" message: Started container splitter-graph-e04f1 metadata: creationTimestamp: "2026-04-16T18:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:54Z" name: splitter-graph-e04f1-7c7f988c75-9hw7m.18a6ea6c583088ed namespace: kserve-ci-e2e-test resourceVersion: "25050" uid: 5529c20b-d83a-4bdd-abd8-8036e5982902 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e04f1} kind: Pod name: splitter-graph-e04f1-7c7f988c75-9hw7m namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: a3fdb99e-7095-49cd-97d7-f6c00152f7d2 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Stopping container splitter-graph-e04f1 metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:03Z" name: splitter-graph-e04f1-7c7f988c75-9hw7m.18a6ea6e6f9a2cb7 namespace: kserve-ci-e2e-test resourceVersion: "25156" uid: 1ffaee5f-3ae7-4601-84fd-3b41a3898e3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:41:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e04f1} kind: Pod name: splitter-graph-e04f1-7c7f988c75-9hw7m namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: a3fdb99e-7095-49cd-97d7-f6c00152f7d2 kind: Event lastTimestamp: "2026-04-16T18:41:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:30Z" name: splitter-graph-e04f1-7c7f988c75-9hw7m.18a6ea6f042958b6 namespace: kserve-ci-e2e-test resourceVersion: "25502" uid: 1a3bb870-29c8-4ace-8273-71a162972bb5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-e04f1-7c7f988c75 namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: af6438cd-f705-408f-bb68-dc1b71f25c42 kind: Event lastTimestamp: "2026-04-16T18:40:52Z" message: 'Created pod: splitter-graph-e04f1-7c7f988c75-9hw7m' metadata: creationTimestamp: "2026-04-16T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:52Z" name: splitter-graph-e04f1-7c7f988c75.18a6ea6c15b6ea43 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 18c270fa-5043-4922-a66c-3dd08a443399 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: 41ba2566-23cf-4586-83b9-97c13e6feecd kind: Event lastTimestamp: "2026-04-16T18:40:52Z" message: Scaled up replica set splitter-graph-e04f1-7c7f988c75 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:52Z" name: splitter-graph-e04f1.18a6ea6c151c863a namespace: kserve-ci-e2e-test resourceVersion: "25018" uid: a54b0f82-0607-4c3f-a723-e23e86644420 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "25032" uid: a2744743-c5ea-4a98-b01a-a2e746302d0c kind: Event lastTimestamp: "2026-04-16T18:40:53Z" message: InferenceGraph [splitter-graph-e04f1] is Ready metadata: creationTimestamp: "2026-04-16T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:53Z" name: splitter-graph-e04f1.18a6ea6c1b00646e namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: 49b7c483-b827-465d-91d9-d245eca9d753 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "25008" uid: a2744743-c5ea-4a98-b01a-a2e746302d0c kind: Event lastTimestamp: "2026-04-16T18:40:53Z" message: 'Failed to update status for InferenceGraph "splitter-graph-e04f1": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-e04f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:53Z" name: splitter-graph-e04f1.18a6ea6c1d5fe5b2 namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: f7464365-c191-4e55-9faf-aa063bdf3e91 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "25008" uid: a2744743-c5ea-4a98-b01a-a2e746302d0c kind: Event lastTimestamp: "2026-04-16T18:40:53Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-e04f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:53Z" name: splitter-graph-e04f1.18a6ea6c1d6031e7 namespace: kserve-ci-e2e-test resourceVersion: "25036" uid: abec8021-aa1e-4b21-9001-e390fac4fed6 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:19:32.537593Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: e3228849-009a-4d99-a71b-830ff0834674 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw.18a6e941f50c30a6 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: fe6aeb30-5dd7-4dd0-b170-36fe526b83c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: e3228849-009a-4d99-a71b-830ff0834674 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw.18a6e941fd001a8e namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 85fe333b-1a78-43d2-a91e-da7ff94e8883 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: e3228849-009a-4d99-a71b-830ff0834674 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw.18a6e941fe4fe70e namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: ba630555-4aa4-44c9-a800-3787eb22194e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: e3228849-009a-4d99-a71b-830ff0834674 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw.18a6e94206a9c646 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: 88510964-1ac8-4105-bb99-f3c9956a9840 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: e3228849-009a-4d99-a71b-830ff0834674 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw.18a6e942072ff3aa namespace: kserve-ci-e2e-test resourceVersion: "14534" uid: 36c7666e-c0ed-464b-973d-3da0c70797e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:19:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: e3228849-009a-4d99-a71b-830ff0834674 kind: Event lastTimestamp: "2026-04-16T18:20:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:19:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:14Z" name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw.18a6e94242f75973 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 010e3cf6-7717-4848-a3c3-34e6d62bcec1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: e3228849-009a-4d99-a71b-830ff0834674 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw.18a6e9c5804f9b1c namespace: kserve-ci-e2e-test resourceVersion: "18904" uid: 3a2f3f26-2294-48fc-80b0-ae8673bd60a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-14a0c-predictor-6fd9f9f998 namespace: kserve-ci-e2e-test resourceVersion: "14464" uid: 9f1bf0f5-ced0-4ca5-8479-1245d0d2e194 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'Created pod: success-200-isvc-14a0c-predictor-6fd9f9f998-6gdkw' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-14a0c-predictor-6fd9f9f998.18a6e941f4958a1b namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b54fef18-6ba7-4a8b-8584-4e619d2d8ed2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: bcd0dcbe-340e-4fc5-bcbe-de06dda2c2ee kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Scaled up replica set success-200-isvc-14a0c-predictor-6fd9f9f998 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-14a0c-predictor.18a6e941f2e31b8c namespace: kserve-ci-e2e-test resourceVersion: "14468" uid: e5996f93-374a-4324-8b50-62859db354b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:19:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: 208d9d0b-06a3-40bf-969f-27813d52e968 kind: Event lastTimestamp: "2026-04-16T18:20:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:02Z" name: success-200-isvc-14a0c-predictor.18a6e94571ef298a namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: a8f66aeb-3b42-489a-a620-6706463132de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:19:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: 208d9d0b-06a3-40bf-969f-27813d52e968 kind: Event lastTimestamp: "2026-04-16T18:20:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:02Z" name: success-200-isvc-14a0c-predictor.18a6e945726f9d4f namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 0d1d3836-4703-4c9d-a320-299c676d29ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:20:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: 208d9d0b-06a3-40bf-969f-27813d52e968 kind: Event lastTimestamp: "2026-04-16T18:20:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:47Z" name: success-200-isvc-14a0c-predictor.18a6e94c6ffcfb1e namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 375942fc-7d62-42f1-9966-3cba972a8996 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:20:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14a0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: 208d9d0b-06a3-40bf-969f-27813d52e968 kind: Event lastTimestamp: "2026-04-16T18:20:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:47Z" name: success-200-isvc-14a0c-predictor.18a6e94c70670f9e namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 7ac7bee5-b432-40d7-941c-4841a747fe17 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14a0c namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 0d57ae9d-b342-41da-b3df-f2e9aa1cb7ad kind: Event lastTimestamp: "2026-04-16T18:20:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:20:24Z" name: success-200-isvc-14a0c.18a6e941f2e5f115 namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: f3b0b5de-6e1f-4c59-9954-f24082f86edd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14a0c namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 0d57ae9d-b342-41da-b3df-f2e9aa1cb7ad kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'Failed to update status for InferenceService "success-200-isvc-14a0c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-14a0c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-14a0c.18a6e941f3666e86 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: e48af478-bed6-4821-9e69-186e51c12b9e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14a0c namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 0d57ae9d-b342-41da-b3df-f2e9aa1cb7ad kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-14a0c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-14a0c.18a6e941f366b725 namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: 1e0fa7c8-a743-44ed-bda6-68a2891cff8f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14a0c namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 0d57ae9d-b342-41da-b3df-f2e9aa1cb7ad kind: Event lastTimestamp: "2026-04-16T18:20:24Z" message: InferenceService [success-200-isvc-14a0c] is Ready metadata: creationTimestamp: "2026-04-16T18:20:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:20:24Z" name: success-200-isvc-14a0c.18a6e94e26f5e29e namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 45ee71b9-d1a0-47bc-8767-216294a8e35a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:41:03.274610Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: 240de76b-805e-45c1-a36a-5c49c583f087 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:41:03Z" name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r.18a6ea6e7af7fe53 namespace: kserve-ci-e2e-test resourceVersion: "25205" uid: f3480f22-c9d4-4fe1-9c5c-a132efddc708 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r namespace: kserve-ci-e2e-test resourceVersion: "25211" uid: 240de76b-805e-45c1-a36a-5c49c583f087 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:41:03Z" name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r.18a6ea6e834a0a17 namespace: kserve-ci-e2e-test resourceVersion: "25236" uid: 78c7f4eb-bf53-45f5-8cde-9053b2a60836 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 240de76b-805e-45c1-a36a-5c49c583f087 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:03Z" name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r.18a6ea6e84d4a4a1 namespace: kserve-ci-e2e-test resourceVersion: "25238" uid: af1c7c96-5f51-44eb-a74c-734831e04921 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 240de76b-805e-45c1-a36a-5c49c583f087 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:03Z" name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r.18a6ea6e8a7e720a namespace: kserve-ci-e2e-test resourceVersion: "25246" uid: 48c4a10a-a19a-42c6-9520-16eac2a4e666 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 240de76b-805e-45c1-a36a-5c49c583f087 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:03Z" name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r.18a6ea6e8b052c32 namespace: kserve-ci-e2e-test resourceVersion: "25248" uid: 79814460-c18d-4657-a9ef-10af8d98a755 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 240de76b-805e-45c1-a36a-5c49c583f087 kind: Event lastTimestamp: "2026-04-16T18:41:45Z" message: 'Readiness probe failed: dial tcp 10.133.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:45Z" name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r.18a6ea6eabae33d9 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: df2d548f-35aa-48fb-a73d-a7b51d790aef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 240de76b-805e-45c1-a36a-5c49c583f087 kind: Event lastTimestamp: "2026-04-16T18:50:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:50:28Z" name: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r.18a6eaf2017aac42 namespace: kserve-ci-e2e-test resourceVersion: "29658" uid: c0eee4b6-384c-472c-b085-e2f8d41071b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1ac5a-predictor-67ff7bfc58 namespace: kserve-ci-e2e-test resourceVersion: "25180" uid: 82a71c82-17da-4236-8eb6-a83fbec1b09e kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: 'Created pod: success-200-isvc-1ac5a-predictor-67ff7bfc58-mgh7r' metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:03Z" name: success-200-isvc-1ac5a-predictor-67ff7bfc58.18a6ea6e7a06fc3a namespace: kserve-ci-e2e-test resourceVersion: "25200" uid: 62d530eb-e619-4e8d-82d2-1858e92ca13d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: e5dcddaa-2ce5-4396-8da2-23e9f6c32a71 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Scaled up replica set success-200-isvc-1ac5a-predictor-67ff7bfc58 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:03Z" name: success-200-isvc-1ac5a-predictor.18a6ea6e775ae11d namespace: kserve-ci-e2e-test resourceVersion: "25187" uid: 9ef1cc05-91e8-431b-80dd-5547b12e9814 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:41:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25179" uid: 46d1033e-a7e0-472b-adfe-190a1f0b8595 kind: Event lastTimestamp: "2026-04-16T18:41:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:33Z" name: success-200-isvc-1ac5a-predictor.18a6ea71f5a53314 namespace: kserve-ci-e2e-test resourceVersion: "25524" uid: 23155970-c834-464c-afdc-bb95e3cc7787 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:41:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25179" uid: 46d1033e-a7e0-472b-adfe-190a1f0b8595 kind: Event lastTimestamp: "2026-04-16T18:41:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:33Z" name: success-200-isvc-1ac5a-predictor.18a6ea71f6053490 namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: 82b1b56d-c7fc-4513-9913-5c6addf71073 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:41:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25390" uid: 46d1033e-a7e0-472b-adfe-190a1f0b8595 kind: Event lastTimestamp: "2026-04-16T18:42:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:18Z" name: success-200-isvc-1ac5a-predictor.18a6ea78f399f2cb namespace: kserve-ci-e2e-test resourceVersion: "25963" uid: 28c44b1d-05be-449b-9f63-a531a18eb1ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:41:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25390" uid: 46d1033e-a7e0-472b-adfe-190a1f0b8595 kind: Event lastTimestamp: "2026-04-16T18:42:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:18Z" name: success-200-isvc-1ac5a-predictor.18a6ea78f410583e namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: a2118444-c13c-4d45-a952-38ffe2b3cbb7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25159" uid: f05d0b3f-b333-4de0-ad80-631cea60a746 kind: Event lastTimestamp: "2026-04-16T18:41:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:55Z" name: success-200-isvc-1ac5a.18a6ea6e76f09db6 namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: 51a3f655-4371-44f7-b26d-8f4e30b51c39 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ac5a namespace: kserve-ci-e2e-test resourceVersion: "25749" uid: f05d0b3f-b333-4de0-ad80-631cea60a746 kind: Event lastTimestamp: "2026-04-16T18:41:55Z" message: InferenceService [success-200-isvc-1ac5a] is Ready metadata: creationTimestamp: "2026-04-16T18:41:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:55Z" name: success-200-isvc-1ac5a.18a6ea7a91c49f34 namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: 4f5a7c79-97e6-4f5a-9b11-b45f2323fa2f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:18:02.307971Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-47091-predictor-7c875766fd-97c5v namespace: kserve-ci-e2e-test resourceVersion: "13459" uid: 69cd51b7-b70f-457d-bd3a-fca57bdb2b2a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-47091-predictor-7c875766fd-97c5v to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:18:02Z" name: success-200-isvc-47091-predictor-7c875766fd-97c5v.18a6e92cf2f16d43 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: 83f76612-2748-4bbb-b214-e55d57fdba73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-47091-predictor-7c875766fd-97c5v namespace: kserve-ci-e2e-test resourceVersion: "13467" uid: 69cd51b7-b70f-457d-bd3a-fca57bdb2b2a kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:18:02Z" name: success-200-isvc-47091-predictor-7c875766fd-97c5v.18a6e92cfa932737 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: bbe971d4-342d-4404-9588-6f27cd9b3de6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-47091-predictor-7c875766fd-97c5v namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 69cd51b7-b70f-457d-bd3a-fca57bdb2b2a kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:02Z" name: success-200-isvc-47091-predictor-7c875766fd-97c5v.18a6e92cfc069dae namespace: kserve-ci-e2e-test resourceVersion: "13508" uid: 5b45eb98-070d-48b5-afc8-4fbb9ad91e51 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-47091-predictor-7c875766fd-97c5v namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 69cd51b7-b70f-457d-bd3a-fca57bdb2b2a kind: Event lastTimestamp: "2026-04-16T18:18:19Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" in 16.867s (16.867s including waiting). Image size: 1332284013 bytes.' metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:19Z" name: success-200-isvc-47091-predictor-7c875766fd-97c5v.18a6e930e9674fdd namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: d22ed484-3c3b-49c5-96fd-e5784355478e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-47091-predictor-7c875766fd-97c5v namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 69cd51b7-b70f-457d-bd3a-fca57bdb2b2a kind: Event lastTimestamp: "2026-04-16T18:18:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:19Z" name: success-200-isvc-47091-predictor-7c875766fd-97c5v.18a6e930ef8eed06 namespace: kserve-ci-e2e-test resourceVersion: "13742" uid: 57137270-4041-4e86-a3f6-63475dbb640f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-47091-predictor-7c875766fd-97c5v namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 69cd51b7-b70f-457d-bd3a-fca57bdb2b2a kind: Event lastTimestamp: "2026-04-16T18:18:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:19Z" name: success-200-isvc-47091-predictor-7c875766fd-97c5v.18a6e930f004b1b8 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: 49a1f7a5-4c7e-49f7-88f9-127a10b1b5e7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-47091-predictor-7c875766fd-97c5v namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 69cd51b7-b70f-457d-bd3a-fca57bdb2b2a kind: Event lastTimestamp: "2026-04-16T18:19:00Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:00Z" name: success-200-isvc-47091-predictor-7c875766fd-97c5v.18a6e930f8d8abbd namespace: kserve-ci-e2e-test resourceVersion: "14100" uid: 5e83f821-2ead-4350-850c-9f908acca0b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-47091-predictor-7c875766fd-97c5v namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 69cd51b7-b70f-457d-bd3a-fca57bdb2b2a kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-47091-predictor-7c875766fd-97c5v.18a6e941f3b5bae0 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: aa7d7dff-0bc9-43c0-a0df-13a495dbb986 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-47091-predictor-7c875766fd namespace: kserve-ci-e2e-test resourceVersion: "13453" uid: 57830dac-c1f4-4e5b-a1ef-3b2c9892a188 kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: 'Created pod: success-200-isvc-47091-predictor-7c875766fd-97c5v' metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: success-200-isvc-47091-predictor-7c875766fd.18a6e92cf275e272 namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: 3996bea8-699f-4d17-bcff-51ae5641a066 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13451" uid: b485c373-809f-4e8b-a2b3-8ebe06fe5d34 kind: Event lastTimestamp: "2026-04-16T18:18:02Z" message: Scaled up replica set success-200-isvc-47091-predictor-7c875766fd from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:02Z" name: success-200-isvc-47091-predictor.18a6e92cf1391127 namespace: kserve-ci-e2e-test resourceVersion: "13455" uid: b1e04891-4145-41dc-9654-fa5a2253b39c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: b27a9b71-44ae-49e0-b9b9-a3480d2cb1db kind: Event lastTimestamp: "2026-04-16T18:18:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:32Z" name: success-200-isvc-47091-predictor.18a6e9306fb94e3e namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: 18a34ca4-3c81-468e-ba18-0cf7c5415b00 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:18:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: b27a9b71-44ae-49e0-b9b9-a3480d2cb1db kind: Event lastTimestamp: "2026-04-16T18:18:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:32Z" name: success-200-isvc-47091-predictor.18a6e9307020a95a namespace: kserve-ci-e2e-test resourceVersion: "13872" uid: 9928062c-33a1-44c9-92d7-f3541850ba84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:18:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13711" uid: b27a9b71-44ae-49e0-b9b9-a3480d2cb1db kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-47091-predictor.18a6e9376da60d4b namespace: kserve-ci-e2e-test resourceVersion: "14423" uid: 1f12671e-feb3-4508-8b69-d7573baf931c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:18:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-47091-predictor namespace: kserve-ci-e2e-test resourceVersion: "13711" uid: b27a9b71-44ae-49e0-b9b9-a3480d2cb1db kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:32Z" name: success-200-isvc-47091-predictor.18a6e9376e08e203 namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: f9d66fe9-34d1-4482-a4ec-fed4095e78b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:18:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-47091 namespace: kserve-ci-e2e-test resourceVersion: "13443" uid: c00d5d3b-c80f-45e4-99b7-252e406f5275 kind: Event lastTimestamp: "2026-04-16T18:19:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:10Z" name: success-200-isvc-47091.18a6e92cf1114377 namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: 83da6b7d-eb5b-4496-94e1-41af9b0f9554 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-47091 namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: c00d5d3b-c80f-45e4-99b7-252e406f5275 kind: Event lastTimestamp: "2026-04-16T18:19:10Z" message: InferenceService [success-200-isvc-47091] is Ready metadata: creationTimestamp: "2026-04-16T18:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:10Z" name: success-200-isvc-47091.18a6e93cdd4e20b5 namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 5455d724-55d4-464b-90ce-b9f3ec0ff7df reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:20:22.435330Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 8e2fad16-84cf-4db7-ba08-e7ca4c7e3b27 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-68d6a-predictor-56b57ff594-f4qdx to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:20:22Z" name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx.18a6e94d932f2770 namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 4333891f-6549-4fcc-8a4b-b194d0f26ed9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 8e2fad16-84cf-4db7-ba08-e7ca4c7e3b27 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:20:22Z" name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx.18a6e94d9b81dc7c namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: ada61c07-8a53-4538-90b8-fe66bd858ba2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 8e2fad16-84cf-4db7-ba08-e7ca4c7e3b27 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx.18a6e94d9cb28e19 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: 82973cf1-aa8c-4b8d-9934-f666394e9cfa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 8e2fad16-84cf-4db7-ba08-e7ca4c7e3b27 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx.18a6e94da4dcddfb namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 98f0d6fe-f6f2-4f7c-a047-f03930dd2c40 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 8e2fad16-84cf-4db7-ba08-e7ca4c7e3b27 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:22Z" name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx.18a6e94da5515d1a namespace: kserve-ci-e2e-test resourceVersion: "15144" uid: 8f448177-b8b7-4c3b-bba9-adc5075b427b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:20:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 8e2fad16-84cf-4db7-ba08-e7ca4c7e3b27 kind: Event lastTimestamp: "2026-04-16T18:21:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:20:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:21:04Z" name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx.18a6e94db5a909d8 namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 783416a6-5c02-4af4-be54-93640aba9478 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 8e2fad16-84cf-4db7-ba08-e7ca4c7e3b27 kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx.18a6e9d120885961 namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 6b54838d-4a9c-40a5-8e1d-615fe9a53c14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-68d6a-predictor-56b57ff594 namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: d4840a58-e30d-4b53-a60c-684382fb3553 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: 'Created pod: success-200-isvc-68d6a-predictor-56b57ff594-f4qdx' metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:22Z" name: success-200-isvc-68d6a-predictor-56b57ff594.18a6e94d929718e3 namespace: kserve-ci-e2e-test resourceVersion: "15087" uid: 92eac8bc-0a61-48aa-81e0-523dc9561478 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: faae2276-2b13-4f24-8c5c-7c16d23043e3 kind: Event lastTimestamp: "2026-04-16T18:20:22Z" message: Scaled up replica set success-200-isvc-68d6a-predictor-56b57ff594 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:22Z" name: success-200-isvc-68d6a-predictor.18a6e94d910d77ce namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: a1573c81-7e31-416b-b3b3-dc24ccae4297 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 9f6750e0-9e77-4a3d-9e8e-93e4f9cd1ec6 kind: Event lastTimestamp: "2026-04-16T18:20:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:37Z" name: success-200-isvc-68d6a-predictor.18a6e9510fb46dfe namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: a52dd08e-7e5b-4f0a-afc5-ea8dbcfc51b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 9f6750e0-9e77-4a3d-9e8e-93e4f9cd1ec6 kind: Event lastTimestamp: "2026-04-16T18:20:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:37Z" name: success-200-isvc-68d6a-predictor.18a6e951102ba35a namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: affbd271-3e82-484a-af19-9a7ce7945c9e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:20:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 9f6750e0-9e77-4a3d-9e8e-93e4f9cd1ec6 kind: Event lastTimestamp: "2026-04-16T18:21:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:52Z" name: success-200-isvc-68d6a-predictor.18a6e9548f455887 namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 82fa2c4c-9ef4-457a-a9b3-69703a2c40af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:20:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68d6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 9f6750e0-9e77-4a3d-9e8e-93e4f9cd1ec6 kind: Event lastTimestamp: "2026-04-16T18:21:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:52Z" name: success-200-isvc-68d6a-predictor.18a6e9548fb49683 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 2d6f3fe1-9627-4768-a516-fdc4bae47b82 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:20:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 0abfe254-2cc0-4d25-a64c-bb1614470660 kind: Event lastTimestamp: "2026-04-16T18:21:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:21:14Z" name: success-200-isvc-68d6a.18a6e94d90ef9329 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 6c477bf5-954a-4107-be01-acf0a63d8e04 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:21:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68d6a namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 0abfe254-2cc0-4d25-a64c-bb1614470660 kind: Event lastTimestamp: "2026-04-16T18:21:14Z" message: InferenceService [success-200-isvc-68d6a] is Ready metadata: creationTimestamp: "2026-04-16T18:21:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:21:14Z" name: success-200-isvc-68d6a.18a6e9599ba43388 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: 9fe26869-83bb-4265-b011-1010630e6ee3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:30:18.070403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 176a5137-9f81-42d9-9bae-968764c70863 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bac14-predictor-6df98975fd-g7z7n to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:30:18Z" name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n.18a6e9d841cd2825 namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 80bbf296-c443-4654-9a78-98de6ac16e62 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: 176a5137-9f81-42d9-9bae-968764c70863 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:30:18Z" name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n.18a6e9d84a73fcac namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: deaddb5d-149e-4199-969e-2faeab4bf7c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 176a5137-9f81-42d9-9bae-968764c70863 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:18Z" name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n.18a6e9d84c94f39c namespace: kserve-ci-e2e-test resourceVersion: "19895" uid: 2cd2f4b1-6fcd-4806-9601-c419e232dbb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 176a5137-9f81-42d9-9bae-968764c70863 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:18Z" name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n.18a6e9d851a44cfd namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: c40e9702-41c7-4621-a3c6-560f20995842 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 176a5137-9f81-42d9-9bae-968764c70863 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:18Z" name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n.18a6e9d852352e61 namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: d28f82a3-06a7-460e-a340-1d5b09edca0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 176a5137-9f81-42d9-9bae-968764c70863 kind: Event lastTimestamp: "2026-04-16T18:30:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:59Z" name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n.18a6e9d876ff0747 namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: c7d443e5-c6b1-4510-aa0d-da62adbce4e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 176a5137-9f81-42d9-9bae-968764c70863 kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:42Z" name: success-200-isvc-bac14-predictor-6df98975fd-g7z7n.18a6ea5bbf81b8bd namespace: kserve-ci-e2e-test resourceVersion: "24239" uid: 6d9de714-e94b-4565-aeed-8e22b10e1938 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bac14-predictor-6df98975fd namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: 9320bbbf-b551-4464-af2b-2b46f3993fba kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: 'Created pod: success-200-isvc-bac14-predictor-6df98975fd-g7z7n' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:18Z" name: success-200-isvc-bac14-predictor-6df98975fd.18a6e9d840f7eaee namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: d5f9d03c-a174-4fdd-a96d-fbeb1c595d53 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 21baf43f-e5fb-44f9-841a-ef67b9233be3 kind: Event lastTimestamp: "2026-04-16T18:30:17Z" message: Scaled up replica set success-200-isvc-bac14-predictor-6df98975fd from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:17Z" name: success-200-isvc-bac14-predictor.18a6e9d83c54c3fd namespace: kserve-ci-e2e-test resourceVersion: "19838" uid: 3cb91ef8-c864-48cc-a1d5-b139d02d802f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: ba80019f-b4f7-4cac-bfc0-86b7ea76cb04 kind: Event lastTimestamp: "2026-04-16T18:30:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:32Z" name: success-200-isvc-bac14-predictor.18a6e9dbbb469983 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 26f2b05d-f1a7-42c6-9f00-9d11ca6b2867 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: ba80019f-b4f7-4cac-bfc0-86b7ea76cb04 kind: Event lastTimestamp: "2026-04-16T18:30:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:33Z" name: success-200-isvc-bac14-predictor.18a6e9dbbc0fc33d namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 42b1cdba-b3f0-43ec-9280-da17acc71b75 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:30:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: ba80019f-b4f7-4cac-bfc0-86b7ea76cb04 kind: Event lastTimestamp: "2026-04-16T18:31:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:33Z" name: success-200-isvc-bac14-predictor.18a6e9df3d58a42b namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 150ca709-fde5-43f2-b821-8d92f25e6388 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:30:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bac14-predictor namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: ba80019f-b4f7-4cac-bfc0-86b7ea76cb04 kind: Event lastTimestamp: "2026-04-16T18:31:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:33Z" name: success-200-isvc-bac14-predictor.18a6e9df3de7f5d6 namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: 86361bd4-bf0e-49be-b654-2db180eece80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:30:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bac14 namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: bf55fd05-7498-4c6c-a26a-b92b73a41621 kind: Event lastTimestamp: "2026-04-16T18:31:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:31:10Z" name: success-200-isvc-bac14.18a6e9d83c6c9f96 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 2354fc8e-6e64-4629-aa26-82ed6515d42d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bac14 namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: bf55fd05-7498-4c6c-a26a-b92b73a41621 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: 'Failed to update status for InferenceService "success-200-isvc-bac14": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bac14": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:18Z" name: success-200-isvc-bac14.18a6e9d83dfb7193 namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: f78a6986-5d02-4821-8ba3-6f89d110d099 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bac14 namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: bf55fd05-7498-4c6c-a26a-b92b73a41621 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bac14": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:18Z" name: success-200-isvc-bac14.18a6e9d83dfbcd8e namespace: kserve-ci-e2e-test resourceVersion: "19847" uid: df82f342-66da-42df-a33b-0958bc680f67 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bac14 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: bf55fd05-7498-4c6c-a26a-b92b73a41621 kind: Event lastTimestamp: "2026-04-16T18:31:10Z" message: InferenceService [success-200-isvc-bac14] is Ready metadata: creationTimestamp: "2026-04-16T18:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:31:10Z" name: success-200-isvc-bac14.18a6e9e4681d43c2 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: b9bfe8bf-6d66-47c4-a086-c788da266c6e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:29:47.482517Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4 namespace: kserve-ci-e2e-test resourceVersion: "19411" uid: 043d90df-836c-46a3-9145-e88f9c1e786f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c659a-predictor-587d4db8d4-kzxw4 to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4.18a6e9d1229f02e4 namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: 353c15d0-8936-41e1-92a5-fb68844d4102 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4 namespace: kserve-ci-e2e-test resourceVersion: "19421" uid: 043d90df-836c-46a3-9145-e88f9c1e786f kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4.18a6e9d12b081bc4 namespace: kserve-ci-e2e-test resourceVersion: "19459" uid: 313f5ff1-30fe-42cb-b4fc-d0c7c8534eaf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 043d90df-836c-46a3-9145-e88f9c1e786f kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4.18a6e9d12c94dbfe namespace: kserve-ci-e2e-test resourceVersion: "19461" uid: 9d3bf160-c2d3-4acc-a66e-081efb790f6a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 043d90df-836c-46a3-9145-e88f9c1e786f kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4.18a6e9d131730e2c namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 97d626cd-c8b1-4a42-b3fb-681f3a105352 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 043d90df-836c-46a3-9145-e88f9c1e786f kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4.18a6e9d131f4f293 namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: 1401b621-fc5d-4974-8ca1-b419541deb2e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 043d90df-836c-46a3-9145-e88f9c1e786f kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4.18a6e9d138d09c4d namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: f03c9895-1459-48f8-a8eb-4d83ef970902 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 043d90df-836c-46a3-9145-e88f9c1e786f kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4.18a6e9e3e714df7c namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 9949c97c-331f-48c9-933c-dfd1bbc8657c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c659a-predictor-587d4db8d4 namespace: kserve-ci-e2e-test resourceVersion: "19397" uid: e4dc9a9b-9b18-4ff7-89b2-0f162caed545 kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: 'Created pod: success-200-isvc-c659a-predictor-587d4db8d4-kzxw4' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-c659a-predictor-587d4db8d4.18a6e9d121f3f59d namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: 19eecc67-24f5-439c-930b-72afd784e3d1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: 9b0049af-6a9f-4aff-a01f-380b0c05e6f7 kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: Scaled up replica set success-200-isvc-c659a-predictor-587d4db8d4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-c659a-predictor.18a6e9d11f423ada namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: df5592b6-c62e-4b16-8721-f471704a6bac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: bc52ad74-9f10-4dab-b2a0-f56ee7967b0b kind: Event lastTimestamp: "2026-04-16T18:30:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:17Z" name: success-200-isvc-c659a-predictor.18a6e9d49e7ffc6a namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 41a7d5ee-75cc-4f0c-97e6-4a0be0b5faae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: bc52ad74-9f10-4dab-b2a0-f56ee7967b0b kind: Event lastTimestamp: "2026-04-16T18:30:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:17Z" name: success-200-isvc-c659a-predictor.18a6e9d49f9f1333 namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 064cb614-e671-4a09-8403-af108d2bd76b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:30:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: bc52ad74-9f10-4dab-b2a0-f56ee7967b0b kind: Event lastTimestamp: "2026-04-16T18:31:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:02Z" name: success-200-isvc-c659a-predictor.18a6e9db9ea86c67 namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: 3fa07070-eec0-4843-b371-e01b38dbfbdc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:30:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c659a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: bc52ad74-9f10-4dab-b2a0-f56ee7967b0b kind: Event lastTimestamp: "2026-04-16T18:31:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:02Z" name: success-200-isvc-c659a-predictor.18a6e9db9f300fad namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 13d93e28-c5a6-4f5b-b5ce-540871198b42 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c659a namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: 327594cc-4772-47c8-a9cb-3a12b748f00b kind: Event lastTimestamp: "2026-04-16T18:30:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:39Z" name: success-200-isvc-c659a.18a6e9d11f4a3abd namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: ff85e4e1-c433-466b-afb7-974c6a46040f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c659a namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: 327594cc-4772-47c8-a9cb-3a12b748f00b kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-c659a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c659a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-c659a.18a6e9d11fbd3885 namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: 0b6dfc95-e5ba-45d5-a427-c5c0c67f89cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c659a namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: 327594cc-4772-47c8-a9cb-3a12b748f00b kind: Event lastTimestamp: "2026-04-16T18:29:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c659a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:47Z" name: success-200-isvc-c659a.18a6e9d11fbd8cc8 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 555f75e7-c952-46f3-84cb-a9c3bfe9f2d8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c659a namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 327594cc-4772-47c8-a9cb-3a12b748f00b kind: Event lastTimestamp: "2026-04-16T18:30:38Z" message: InferenceService [success-200-isvc-c659a] is Ready metadata: creationTimestamp: "2026-04-16T18:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:30:38Z" name: success-200-isvc-c659a.18a6e9dd20decb99 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: d217bf26-b11c-4fad-bb4c-f71db3b059f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:57.581285Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg namespace: kserve-ci-e2e-test resourceVersion: "18922" uid: 8020656a-80c3-4415-9248-4c823e247e60 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cc45f-predictor-59db55445d-6kzqg to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg.18a6e9c58446af64 namespace: kserve-ci-e2e-test resourceVersion: "18933" uid: c60f4f41-cd1b-4675-8237-fac60cfe9204 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: 8020656a-80c3-4415-9248-4c823e247e60 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg.18a6e9c58d0b0001 namespace: kserve-ci-e2e-test resourceVersion: "18961" uid: 62dbbaa4-4f72-4c93-871f-893249cf53fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: 8020656a-80c3-4415-9248-4c823e247e60 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg.18a6e9c58e54fb77 namespace: kserve-ci-e2e-test resourceVersion: "18963" uid: 3f0b403c-eee3-49d0-8d7b-ad6c268c088f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: 8020656a-80c3-4415-9248-4c823e247e60 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg.18a6e9c5931e1f54 namespace: kserve-ci-e2e-test resourceVersion: "18970" uid: 7b1c3ac4-1f39-4ede-90be-daf6760288dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: 8020656a-80c3-4415-9248-4c823e247e60 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg.18a6e9c5939cde75 namespace: kserve-ci-e2e-test resourceVersion: "18971" uid: 1e4c2044-6f7d-4a0d-be49-519ee84339af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: 8020656a-80c3-4415-9248-4c823e247e60 kind: Event lastTimestamp: "2026-04-16T18:30:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:19Z" name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg.18a6e9c5c5f0bf82 namespace: kserve-ci-e2e-test resourceVersion: "19937" uid: 5f51528e-63b9-4a7e-a8f8-bad2313be1a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: 8020656a-80c3-4415-9248-4c823e247e60 kind: Event lastTimestamp: "2026-04-16T18:30:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:30:18Z" name: success-200-isvc-cc45f-predictor-59db55445d-6kzqg.18a6e9d8487c6181 namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: d8e59966-4233-483e-aedf-9657dbf76ca5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cc45f-predictor-59db55445d namespace: kserve-ci-e2e-test resourceVersion: "18902" uid: da51125c-9a4e-4e92-9a26-6fabd7affcb3 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: 'Created pod: success-200-isvc-cc45f-predictor-59db55445d-6kzqg' metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-cc45f-predictor-59db55445d.18a6e9c583fda660 namespace: kserve-ci-e2e-test resourceVersion: "18926" uid: ab23dd66-5b6e-4a11-84ea-b3a2b3ea60d1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18899" uid: a9463d3c-f3d1-422c-8875-ae863e0bf262 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Scaled up replica set success-200-isvc-cc45f-predictor-59db55445d from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-cc45f-predictor.18a6e9c58060a591 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: 4ca5d20a-9f8a-4c76-b20b-cd74392aa573 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: 8edccfee-2523-4d6a-bb05-a314a685d69a kind: Event lastTimestamp: "2026-04-16T18:29:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:12Z" name: success-200-isvc-cc45f-predictor.18a6e9c8ff0ab344 namespace: kserve-ci-e2e-test resourceVersion: "19097" uid: 58e2dc33-5d84-4701-9e18-e71d46987dd5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: 8edccfee-2523-4d6a-bb05-a314a685d69a kind: Event lastTimestamp: "2026-04-16T18:29:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:12Z" name: success-200-isvc-cc45f-predictor.18a6e9c8ff6f12f7 namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: 2df847ee-90e6-4f8c-b1ac-f30b7a496eab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:29:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 8edccfee-2523-4d6a-bb05-a314a685d69a kind: Event lastTimestamp: "2026-04-16T18:30:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:12Z" name: success-200-isvc-cc45f-predictor.18a6e9cc7e310a42 namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 95960edf-c914-47d4-8080-9d672975ef25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:29:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cc45f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 8edccfee-2523-4d6a-bb05-a314a685d69a kind: Event lastTimestamp: "2026-04-16T18:30:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:12Z" name: success-200-isvc-cc45f-predictor.18a6e9cc7e9b3306 namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: dc9b0173-cc38-4c1f-a0a2-7db6541dc01b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cc45f namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: cbd62baf-60ab-42e9-8ef4-70edb3b99840 kind: Event lastTimestamp: "2026-04-16T18:29:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:49Z" name: success-200-isvc-cc45f.18a6e9c5803a253b namespace: kserve-ci-e2e-test resourceVersion: "19520" uid: cfb675d5-09d1-4db3-9fac-f5ecda71bb13 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cc45f namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: cbd62baf-60ab-42e9-8ef4-70edb3b99840 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: 'Failed to update status for InferenceService "success-200-isvc-cc45f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cc45f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-cc45f.18a6e9c580beb35d namespace: kserve-ci-e2e-test resourceVersion: "18907" uid: d1ec5c12-cefb-4fc1-be84-1312b6e5c58b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cc45f namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: cbd62baf-60ab-42e9-8ef4-70edb3b99840 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cc45f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:57Z" name: success-200-isvc-cc45f.18a6e9c580bf0ab1 namespace: kserve-ci-e2e-test resourceVersion: "18908" uid: 9f38167b-0de2-44cb-9c76-f40e442e6557 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cc45f namespace: kserve-ci-e2e-test resourceVersion: "19516" uid: cbd62baf-60ab-42e9-8ef4-70edb3b99840 kind: Event lastTimestamp: "2026-04-16T18:29:49Z" message: InferenceService [success-200-isvc-cc45f] is Ready metadata: creationTimestamp: "2026-04-16T18:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:49Z" name: success-200-isvc-cc45f.18a6e9d1ac88ade7 namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 2b571c46-88de-4cc3-be6b-c7a96c69d5e7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:40:32.728574Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv namespace: kserve-ci-e2e-test resourceVersion: "24762" uid: 2e645ea6-f77f-4abb-86cd-c23525ea395e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv.18a6ea675e488421 namespace: kserve-ci-e2e-test resourceVersion: "24766" uid: 2502cac3-439d-4908-928b-303f179c9b97 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv namespace: kserve-ci-e2e-test resourceVersion: "24771" uid: 2e645ea6-f77f-4abb-86cd-c23525ea395e kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv.18a6ea67663e5f6b namespace: kserve-ci-e2e-test resourceVersion: "24799" uid: 7daec372-53c7-4183-81a5-e38bc7bd3fdd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv namespace: kserve-ci-e2e-test resourceVersion: "24764" uid: 2e645ea6-f77f-4abb-86cd-c23525ea395e kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv.18a6ea67679e5dde namespace: kserve-ci-e2e-test resourceVersion: "24803" uid: 144ab7e2-286e-4169-b6fe-0cec85f5d863 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv namespace: kserve-ci-e2e-test resourceVersion: "24764" uid: 2e645ea6-f77f-4abb-86cd-c23525ea395e kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv.18a6ea676b28d1f4 namespace: kserve-ci-e2e-test resourceVersion: "24806" uid: 4f92617d-dfbc-4347-9c37-a68241e66298 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv namespace: kserve-ci-e2e-test resourceVersion: "24764" uid: 2e645ea6-f77f-4abb-86cd-c23525ea395e kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv.18a6ea676ba4ed1e namespace: kserve-ci-e2e-test resourceVersion: "24807" uid: 877bfa00-0258-4319-9d5a-efe564cd1e3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv namespace: kserve-ci-e2e-test resourceVersion: "24764" uid: 2e645ea6-f77f-4abb-86cd-c23525ea395e kind: Event lastTimestamp: "2026-04-16T18:58:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:58:03Z" name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv.18a6ea676d7f676f namespace: kserve-ci-e2e-test resourceVersion: "32858" uid: 4f03590a-9c2f-4e1a-9b96-102840226a22 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv namespace: kserve-ci-e2e-test resourceVersion: "24764" uid: 2e645ea6-f77f-4abb-86cd-c23525ea395e kind: Event lastTimestamp: "2026-04-16T18:58:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:58:02Z" name: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv.18a6eb5bc569a2f6 namespace: kserve-ci-e2e-test resourceVersion: "32806" uid: 9330f05e-619d-422e-a440-67a00a7eadbf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dbc56-predictor-767bcc6cfb namespace: kserve-ci-e2e-test resourceVersion: "24745" uid: a6e9dc8d-5d1b-46da-98cc-accbbf485822 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: 'Created pod: success-200-isvc-dbc56-predictor-767bcc6cfb-qjprv' metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-dbc56-predictor-767bcc6cfb.18a6ea675db9f538 namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 537d9952-78b7-40ae-b2cb-0d13e4a6c157 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: b8472746-a93f-411f-a601-209aedbe609e kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Scaled up replica set success-200-isvc-dbc56-predictor-767bcc6cfb from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-dbc56-predictor.18a6ea675ba6ad3f namespace: kserve-ci-e2e-test resourceVersion: "24748" uid: 228757a2-4b00-48e2-9c72-d04198b26e53 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24743" uid: 5c5d8cb4-bb73-460f-8529-edc4a9e32636 kind: Event lastTimestamp: "2026-04-16T18:40:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:47Z" name: success-200-isvc-dbc56-predictor.18a6ea6ada562d48 namespace: kserve-ci-e2e-test resourceVersion: "24965" uid: 57d92612-4805-449d-98ef-293fb5779e4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24743" uid: 5c5d8cb4-bb73-460f-8529-edc4a9e32636 kind: Event lastTimestamp: "2026-04-16T18:40:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:47Z" name: success-200-isvc-dbc56-predictor.18a6ea6adac287a5 namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: 063551c9-4c6b-4f1d-8eeb-0ebd651c8fb8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:41:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24966" uid: 5c5d8cb4-bb73-460f-8529-edc4a9e32636 kind: Event lastTimestamp: "2026-04-16T18:41:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:41:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:47Z" name: success-200-isvc-dbc56-predictor.18a6ea6e5991f606 namespace: kserve-ci-e2e-test resourceVersion: "25670" uid: 50a9ba35-6963-455b-b2f5-711481aec170 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:41:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dbc56-predictor namespace: kserve-ci-e2e-test resourceVersion: "24966" uid: 5c5d8cb4-bb73-460f-8529-edc4a9e32636 kind: Event lastTimestamp: "2026-04-16T18:41:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:41:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:47Z" name: success-200-isvc-dbc56-predictor.18a6ea6e5a1686a1 namespace: kserve-ci-e2e-test resourceVersion: "25671" uid: 98a6fc8d-8982-4c76-b4b3-ed7c6e930a4d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "24719" uid: 15e45482-6f78-4d43-b492-c0465fe85275 kind: Event lastTimestamp: "2026-04-16T18:41:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:24Z" name: success-200-isvc-dbc56.18a6ea675ba2d1ca namespace: kserve-ci-e2e-test resourceVersion: "25442" uid: cf8057f2-18a9-4ff7-89d7-4f8161dc77fb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "24719" uid: 15e45482-6f78-4d43-b492-c0465fe85275 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: 'Failed to update status for InferenceService "success-200-isvc-dbc56": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dbc56": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-dbc56.18a6ea675c4cac96 namespace: kserve-ci-e2e-test resourceVersion: "24755" uid: 1c04e0f1-2936-49df-874c-9a3dfec67688 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "24719" uid: 15e45482-6f78-4d43-b492-c0465fe85275 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dbc56": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-dbc56.18a6ea675c4cfedb namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: 3619a0d3-a199-4c01-ab23-13c6b5dee3bf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "25440" uid: 15e45482-6f78-4d43-b492-c0465fe85275 kind: Event lastTimestamp: "2026-04-16T18:41:24Z" message: InferenceService [success-200-isvc-dbc56] is Ready metadata: creationTimestamp: "2026-04-16T18:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:24Z" name: success-200-isvc-dbc56.18a6ea7351c49972 namespace: kserve-ci-e2e-test resourceVersion: "25441" uid: 31bdbda2-413c-4712-9192-5c024f2f0d3b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:39:42.874023Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e04f1-predictor-df557c688-bbnbt namespace: kserve-ci-e2e-test resourceVersion: "24254" uid: 7f829679-ab15-4bb0-8524-a33bb5964964 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e04f1-predictor-df557c688-bbnbt to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:39:42Z" name: success-200-isvc-e04f1-predictor-df557c688-bbnbt.18a6ea5bc2b87646 namespace: kserve-ci-e2e-test resourceVersion: "24265" uid: 856f1904-1d8a-4e1d-85a5-4e208d415751 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e04f1-predictor-df557c688-bbnbt namespace: kserve-ci-e2e-test resourceVersion: "24276" uid: 7f829679-ab15-4bb0-8524-a33bb5964964 kind: Event lastTimestamp: "2026-04-16T18:39:43Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:39:43Z" name: success-200-isvc-e04f1-predictor-df557c688-bbnbt.18a6ea5bcb2b499e namespace: kserve-ci-e2e-test resourceVersion: "24294" uid: ba451873-8269-4a34-8484-3964a679f79b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e04f1-predictor-df557c688-bbnbt namespace: kserve-ci-e2e-test resourceVersion: "24260" uid: 7f829679-ab15-4bb0-8524-a33bb5964964 kind: Event lastTimestamp: "2026-04-16T18:39:43Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:43Z" name: success-200-isvc-e04f1-predictor-df557c688-bbnbt.18a6ea5bcc840cc7 namespace: kserve-ci-e2e-test resourceVersion: "24297" uid: b10f84e1-4c99-4025-9f24-7750a2875ffd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e04f1-predictor-df557c688-bbnbt namespace: kserve-ci-e2e-test resourceVersion: "24260" uid: 7f829679-ab15-4bb0-8524-a33bb5964964 kind: Event lastTimestamp: "2026-04-16T18:39:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:43Z" name: success-200-isvc-e04f1-predictor-df557c688-bbnbt.18a6ea5bd214cd62 namespace: kserve-ci-e2e-test resourceVersion: "24305" uid: 5a3fc531-f2dd-411c-a4f8-d9fb9f39e859 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e04f1-predictor-df557c688-bbnbt namespace: kserve-ci-e2e-test resourceVersion: "24260" uid: 7f829679-ab15-4bb0-8524-a33bb5964964 kind: Event lastTimestamp: "2026-04-16T18:39:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:43Z" name: success-200-isvc-e04f1-predictor-df557c688-bbnbt.18a6ea5bd2929bb8 namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: dfcc9f48-9071-4296-b7d8-faced5a921ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e04f1-predictor-df557c688-bbnbt namespace: kserve-ci-e2e-test resourceVersion: "24260" uid: 7f829679-ab15-4bb0-8524-a33bb5964964 kind: Event lastTimestamp: "2026-04-16T18:41:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:04Z" name: success-200-isvc-e04f1-predictor-df557c688-bbnbt.18a6ea5bfa484d9d namespace: kserve-ci-e2e-test resourceVersion: "25274" uid: b1a585ad-f8b2-409a-a90d-649dbabc7f08 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e04f1-predictor-df557c688-bbnbt namespace: kserve-ci-e2e-test resourceVersion: "24260" uid: 7f829679-ab15-4bb0-8524-a33bb5964964 kind: Event lastTimestamp: "2026-04-16T18:41:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:03Z" name: success-200-isvc-e04f1-predictor-df557c688-bbnbt.18a6ea6e79a01644 namespace: kserve-ci-e2e-test resourceVersion: "25199" uid: d10a06bf-7255-4bb1-9cc0-c2295486f3b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e04f1-predictor-df557c688 namespace: kserve-ci-e2e-test resourceVersion: "24234" uid: b90747b2-6799-425b-a7ce-80a681833d87 kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: 'Created pod: success-200-isvc-e04f1-predictor-df557c688-bbnbt' metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:42Z" name: success-200-isvc-e04f1-predictor-df557c688.18a6ea5bc25859a7 namespace: kserve-ci-e2e-test resourceVersion: "24259" uid: 4b8e2c3c-23d8-4893-8dfd-eb5984a3ed57 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: 3eecc8d8-89f0-42ad-857a-f7fbbb2f70d1 kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: Scaled up replica set success-200-isvc-e04f1-predictor-df557c688 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:42Z" name: success-200-isvc-e04f1-predictor.18a6ea5bbf12bddc namespace: kserve-ci-e2e-test resourceVersion: "24236" uid: 0f8a6540-eb5b-4242-8da1-7811b5ecd3fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:39:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24233" uid: 2eb01af5-3233-4935-a4fe-34b518c4843f kind: Event lastTimestamp: "2026-04-16T18:40:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:12Z" name: success-200-isvc-e04f1-predictor.18a6ea5f3dcbc458 namespace: kserve-ci-e2e-test resourceVersion: "24550" uid: c428f11d-8e9a-43ba-a927-fb88e70e0828 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:39:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24233" uid: 2eb01af5-3233-4935-a4fe-34b518c4843f kind: Event lastTimestamp: "2026-04-16T18:40:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:12Z" name: success-200-isvc-e04f1-predictor.18a6ea5f3e2fd278 namespace: kserve-ci-e2e-test resourceVersion: "24551" uid: dd2df295-53ec-472a-b9cb-99679c73941c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:40:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24436" uid: 2eb01af5-3233-4935-a4fe-34b518c4843f kind: Event lastTimestamp: "2026-04-16T18:40:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:57Z" name: success-200-isvc-e04f1-predictor.18a6ea663bd35474 namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 199262e9-f4f2-4b43-a4f4-026ed7edf10d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:40:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e04f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "24436" uid: 2eb01af5-3233-4935-a4fe-34b518c4843f kind: Event lastTimestamp: "2026-04-16T18:40:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:57Z" name: success-200-isvc-e04f1-predictor.18a6ea663c3e68db namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: 42e2d148-e15f-4dd9-a1ba-93c87407bfe7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "24215" uid: 2191d1c2-23d8-4409-90b9-00996a5fb36f kind: Event lastTimestamp: "2026-04-16T18:40:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:34Z" name: success-200-isvc-e04f1.18a6ea5bbf1c980d namespace: kserve-ci-e2e-test resourceVersion: "24858" uid: ca93cbf7-36f4-4fad-8f33-a6e4a8c2c0ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "24215" uid: 2191d1c2-23d8-4409-90b9-00996a5fb36f kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: 'Failed to update status for InferenceService "success-200-isvc-e04f1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e04f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:42Z" name: success-200-isvc-e04f1.18a6ea5bbf8bf43e namespace: kserve-ci-e2e-test resourceVersion: "24240" uid: 89434641-6d73-4b02-8f36-b0a943dcdb08 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "24215" uid: 2191d1c2-23d8-4409-90b9-00996a5fb36f kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e04f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:42Z" name: success-200-isvc-e04f1.18a6ea5bbf8c3fea namespace: kserve-ci-e2e-test resourceVersion: "24241" uid: cd873c2b-6b5d-4802-a988-4938dcff2746 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e04f1 namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 2191d1c2-23d8-4409-90b9-00996a5fb36f kind: Event lastTimestamp: "2026-04-16T18:40:34Z" message: InferenceService [success-200-isvc-e04f1] is Ready metadata: creationTimestamp: "2026-04-16T18:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:34Z" name: success-200-isvc-e04f1.18a6ea67e08031df namespace: kserve-ci-e2e-test resourceVersion: "24855" uid: bf48b5b4-ff9f-47eb-9e47-dbd1e20fc418 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:31:08.040822Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4 namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: 647737af-9671-4afe-8538-e0a4e77b86a2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4 to ip-10-0-143-51.ec2.internal metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:31:08Z" name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4.18a6e9e3e4452ecc namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: 91102d6e-3433-4bc6-9829-f5c9e3ae3e6c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 647737af-9671-4afe-8538-e0a4e77b86a2 kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:31:08Z" name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4.18a6e9e3ebbe5e34 namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 4816d855-d323-4a3a-9511-81de204b5ef4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 647737af-9671-4afe-8538-e0a4e77b86a2 kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4.18a6e9e3f06de341 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 4e461471-6adf-40af-99bb-8cd81a0fe7e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 647737af-9671-4afe-8538-e0a4e77b86a2 kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4.18a6e9e3f59722e8 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: 07a927e8-1981-418f-bd86-c91737459680 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 647737af-9671-4afe-8538-e0a4e77b86a2 kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:08Z" name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4.18a6e9e3f60c2ce2 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: d17b48c0-9217-453c-804f-f12d5dbe9af4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 647737af-9671-4afe-8538-e0a4e77b86a2 kind: Event lastTimestamp: "2026-04-16T18:31:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:31:50Z" name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4.18a6e9e42587882c namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 9a7f69ef-13a5-4678-be90-49625cc6f4d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 647737af-9671-4afe-8538-e0a4e77b86a2 kind: Event lastTimestamp: "2026-04-16T18:40:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:32Z" name: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4.18a6ea675d076986 namespace: kserve-ci-e2e-test resourceVersion: "24761" uid: db2f756d-db54-40e7-853d-f6120a3c8af8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-51.ec2.internal source: component: kubelet host: ip-10-0-143-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e9680-predictor-7bc5c87f48 namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: c41f033d-5d3c-4a10-b9b1-dc0335bd347d kind: Event lastTimestamp: "2026-04-16T18:31:08Z" message: 'Created pod: success-200-isvc-e9680-predictor-7bc5c87f48-dsrc4' metadata: creationTimestamp: "2026-04-16T18:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:08Z" name: success-200-isvc-e9680-predictor-7bc5c87f48.18a6e9e3e2b63a38 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: be8bf2e5-d305-47de-9978-e352eabb0acd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 82c3a4c1-0f53-4364-8bd6-86f2464163a4 kind: Event lastTimestamp: "2026-04-16T18:31:07Z" message: Scaled up replica set success-200-isvc-e9680-predictor-7bc5c87f48 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:07Z" name: success-200-isvc-e9680-predictor.18a6e9e3ddd178aa namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: eef9502d-180c-4897-a172-515ef5f36f09 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: a3a682c2-1047-48a5-b809-ab0f25c39f3b kind: Event lastTimestamp: "2026-04-16T18:31:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:23Z" name: success-200-isvc-e9680-predictor.18a6e9e760bfd682 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 497c6c80-c0d8-48c2-aaca-e8ca298c7c10 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:31:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: a3a682c2-1047-48a5-b809-ab0f25c39f3b kind: Event lastTimestamp: "2026-04-16T18:31:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:31:23Z" name: success-200-isvc-e9680-predictor.18a6e9e761d0a8c3 namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 86169c1c-816d-4c46-bbae-31fa1c127791 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:31:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: a3a682c2-1047-48a5-b809-ab0f25c39f3b kind: Event lastTimestamp: "2026-04-16T18:32:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:38Z" name: success-200-isvc-e9680-predictor.18a6e9eae08e47a6 namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 668d4393-1f1a-45c4-a2a6-7a6d6863d7f0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:31:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e9680-predictor namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: a3a682c2-1047-48a5-b809-ab0f25c39f3b kind: Event lastTimestamp: "2026-04-16T18:32:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:32:38Z" name: success-200-isvc-e9680-predictor.18a6e9eae0f773aa namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: c73b8ced-0b6f-4a62-bcc3-86f740371ce1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:31:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e9680 namespace: kserve-ci-e2e-test resourceVersion: "20408" uid: b677a70b-229f-4415-be91-85ff31cb6c1b kind: Event lastTimestamp: "2026-04-16T18:32:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:32:00Z" name: success-200-isvc-e9680.18a6e9e3deec384a namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 5028607a-42cf-43b5-8111-74b404785baf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:32:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e9680 namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: b677a70b-229f-4415-be91-85ff31cb6c1b kind: Event lastTimestamp: "2026-04-16T18:32:00Z" message: InferenceService [success-200-isvc-e9680] is Ready metadata: creationTimestamp: "2026-04-16T18:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:32:00Z" name: success-200-isvc-e9680.18a6e9f009efd492 namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 9a02ebd2-3de1-4b48-a281-1a1ce69962a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:20:42.633898Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-14a0c-57c765bd5b-44zs6 namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: e1d2a89e-50bc-47ef-a4c9-9e0d6e5a4931 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-14a0c-57c765bd5b-44zs6 to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:20:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:20:42Z" name: switch-graph-14a0c-57c765bd5b-44zs6.18a6e952471ced7c namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: d51a0033-bdc4-464c-9d5d-3a6de74b2e49 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:42Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-14a0c-57c765bd5b-44zs6 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: e1d2a89e-50bc-47ef-a4c9-9e0d6e5a4931 kind: Event lastTimestamp: "2026-04-16T18:20:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-14a0c-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:42Z" name: switch-graph-14a0c-57c765bd5b-44zs6.18a6e95254463abc namespace: kserve-ci-e2e-test resourceVersion: "15376" uid: 9edb47d3-c77a-4b08-b9c7-39166a5d60ee reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:43Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-14a0c-57c765bd5b-44zs6 namespace: kserve-ci-e2e-test resourceVersion: "15366" uid: e1d2a89e-50bc-47ef-a4c9-9e0d6e5a4931 kind: Event lastTimestamp: "2026-04-16T18:20:43Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:20:43Z" name: switch-graph-14a0c-57c765bd5b-44zs6.18a6e95283ea9bfd namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: 61c8fa22-eaba-4d13-b407-43382ec12f15 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-14a0c} kind: Pod name: switch-graph-14a0c-57c765bd5b-44zs6 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: e1d2a89e-50bc-47ef-a4c9-9e0d6e5a4931 kind: Event lastTimestamp: "2026-04-16T18:20:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:43Z" name: switch-graph-14a0c-57c765bd5b-44zs6.18a6e9528524fed3 namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: a956f2ba-aada-4264-a678-b006535ef19a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-14a0c} kind: Pod name: switch-graph-14a0c-57c765bd5b-44zs6 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: e1d2a89e-50bc-47ef-a4c9-9e0d6e5a4931 kind: Event lastTimestamp: "2026-04-16T18:20:43Z" message: 'Created container: switch-graph-14a0c' metadata: creationTimestamp: "2026-04-16T18:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:43Z" name: switch-graph-14a0c-57c765bd5b-44zs6.18a6e95288480f03 namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: eef344dc-f202-430b-b8d3-f58f3d5d71ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-14a0c} kind: Pod name: switch-graph-14a0c-57c765bd5b-44zs6 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: e1d2a89e-50bc-47ef-a4c9-9e0d6e5a4931 kind: Event lastTimestamp: "2026-04-16T18:20:43Z" message: Started container switch-graph-14a0c metadata: creationTimestamp: "2026-04-16T18:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:43Z" name: switch-graph-14a0c-57c765bd5b-44zs6.18a6e95288c21f69 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 778e6e3a-3f39-4947-b102-edbf14a85240 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-14a0c} kind: Pod name: switch-graph-14a0c-57c765bd5b-44zs6 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: e1d2a89e-50bc-47ef-a4c9-9e0d6e5a4931 kind: Event lastTimestamp: "2026-04-16T18:28:57Z" message: Stopping container switch-graph-14a0c metadata: creationTimestamp: "2026-04-16T18:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:57Z" name: switch-graph-14a0c-57c765bd5b-44zs6.18a6e9c579ffa707 namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: f55056da-bae8-4948-a05e-53ad3225f2ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:29:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-14a0c} kind: Pod name: switch-graph-14a0c-57c765bd5b-44zs6 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: e1d2a89e-50bc-47ef-a4c9-9e0d6e5a4931 kind: Event lastTimestamp: "2026-04-16T18:29:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:29:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:25Z" name: switch-graph-14a0c-57c765bd5b-44zs6.18a6e9c616d26a07 namespace: kserve-ci-e2e-test resourceVersion: "19194" uid: 107801ef-4f49-48b9-9dba-6208df25664d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-14a0c-57c765bd5b namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 35159b38-ac1d-4098-a7e0-949b887b9541 kind: Event lastTimestamp: "2026-04-16T18:20:42Z" message: 'Created pod: switch-graph-14a0c-57c765bd5b-44zs6' metadata: creationTimestamp: "2026-04-16T18:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:42Z" name: switch-graph-14a0c-57c765bd5b.18a6e95246c3da5b namespace: kserve-ci-e2e-test resourceVersion: "15360" uid: aab65286-b869-4219-b6b1-f7960ebfa3dd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-14a0c namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 060e0663-80cc-4947-bd4b-0f64d03397a7 kind: Event lastTimestamp: "2026-04-16T18:20:42Z" message: Scaled up replica set switch-graph-14a0c-57c765bd5b from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:20:42Z" name: switch-graph-14a0c.18a6e95245ed61b9 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: 64edfc25-b6c1-48b4-9cd3-a00459bf53b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:20:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-14a0c namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: 0f3f4136-4c98-48c4-a48f-dafa576f8156 kind: Event lastTimestamp: "2026-04-16T18:20:42Z" message: InferenceGraph [switch-graph-14a0c] is Ready metadata: creationTimestamp: "2026-04-16T18:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:20:42Z" name: switch-graph-14a0c.18a6e9524b668360 namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 4c1f9421-3c1d-46f6-9f77-a704aa9bd6c4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:20:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-14a0c namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 85464014-dd42-4e92-9f52-ad620181ac6c kind: Event lastTimestamp: "2026-04-16T18:21:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:12Z" name: switch-graph-14a0c.18a6e955c497c0c5 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 9a3c680e-d43c-4544-941b-a4d2876a149c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:20:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-14a0c namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 85464014-dd42-4e92-9f52-ad620181ac6c kind: Event lastTimestamp: "2026-04-16T18:21:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:21:12Z" name: switch-graph-14a0c.18a6e955c50125c0 namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 6b3f4b4f-fbfa-4b15-bec1-e87eafeb9822 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:19:22.209341Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-47091-595cb5c45c-pcgzw namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: 833b85dc-abb2-43cf-945e-7f65358019e9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-47091-595cb5c45c-pcgzw to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:19:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:19:22Z" name: switch-graph-47091-595cb5c45c-pcgzw.18a6e93f8d6f86a3 namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 59f1abdb-3a03-4166-8e3a-e92f6f5486d0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-47091-595cb5c45c-pcgzw namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: 833b85dc-abb2-43cf-945e-7f65358019e9 kind: Event lastTimestamp: "2026-04-16T18:19:22Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:19:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:19:22Z" name: switch-graph-47091-595cb5c45c-pcgzw.18a6e93fa6ad3aa2 namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: 53fd088a-2744-485c-8791-1e4e5b8ad9ef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-47091} kind: Pod name: switch-graph-47091-595cb5c45c-pcgzw namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 833b85dc-abb2-43cf-945e-7f65358019e9 kind: Event lastTimestamp: "2026-04-16T18:19:22Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" metadata: creationTimestamp: "2026-04-16T18:19:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:22Z" name: switch-graph-47091-595cb5c45c-pcgzw.18a6e93fa7ebd38d namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: edb7410a-c02e-4aee-8ec8-cd2f456150f2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-47091} kind: Pod name: switch-graph-47091-595cb5c45c-pcgzw namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 833b85dc-abb2-43cf-945e-7f65358019e9 kind: Event lastTimestamp: "2026-04-16T18:19:25Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" in 2.628s (2.628s including waiting). Image size: 216233558 bytes.' metadata: creationTimestamp: "2026-04-16T18:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:25Z" name: switch-graph-47091-595cb5c45c-pcgzw.18a6e940449ac942 namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: e5158b5f-afc9-40f6-a12d-15902ccf8c5b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-47091} kind: Pod name: switch-graph-47091-595cb5c45c-pcgzw namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 833b85dc-abb2-43cf-945e-7f65358019e9 kind: Event lastTimestamp: "2026-04-16T18:19:25Z" message: 'Created container: switch-graph-47091' metadata: creationTimestamp: "2026-04-16T18:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:25Z" name: switch-graph-47091-595cb5c45c-pcgzw.18a6e940483d839d namespace: kserve-ci-e2e-test resourceVersion: "14349" uid: 72deabb0-7e67-4ba9-9f6c-c131cf4d6520 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-47091} kind: Pod name: switch-graph-47091-595cb5c45c-pcgzw namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 833b85dc-abb2-43cf-945e-7f65358019e9 kind: Event lastTimestamp: "2026-04-16T18:19:25Z" message: Started container switch-graph-47091 metadata: creationTimestamp: "2026-04-16T18:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:25Z" name: switch-graph-47091-595cb5c45c-pcgzw.18a6e94048b2a158 namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 9075c6f7-7f8d-48fe-8c55-fb6b04728a4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-47091} kind: Pod name: switch-graph-47091-595cb5c45c-pcgzw namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 833b85dc-abb2-43cf-945e-7f65358019e9 kind: Event lastTimestamp: "2026-04-16T18:19:32Z" message: Stopping container switch-graph-47091 metadata: creationTimestamp: "2026-04-16T18:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:19:32Z" name: switch-graph-47091-595cb5c45c-pcgzw.18a6e941ebe8d25c namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: 0ae368c8-54a7-4b74-b55c-b8114b6f6031 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-47091} kind: Pod name: switch-graph-47091-595cb5c45c-pcgzw namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 833b85dc-abb2-43cf-945e-7f65358019e9 kind: Event lastTimestamp: "2026-04-16T18:20:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:20:01Z" name: switch-graph-47091-595cb5c45c-pcgzw.18a6e942f2fed423 namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: c15eb2bc-f2c1-4ac8-a366-0753d12a9df6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-47091-595cb5c45c namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 164e837c-5821-4ef3-b78f-cb1a6bb0c5ad kind: Event lastTimestamp: "2026-04-16T18:19:22Z" message: 'Created pod: switch-graph-47091-595cb5c45c-pcgzw' metadata: creationTimestamp: "2026-04-16T18:19:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:22Z" name: switch-graph-47091-595cb5c45c.18a6e93f8d04f39b namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: 3ef34868-42c5-4773-8ee2-0d4a33437075 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-47091 namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: 952f7dfd-5ff8-45b3-a151-b899117ae478 kind: Event lastTimestamp: "2026-04-16T18:19:22Z" message: Scaled up replica set switch-graph-47091-595cb5c45c from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:19:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:22Z" name: switch-graph-47091.18a6e93f8c35ab9e namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: ba1592d8-ea7f-43d8-acec-63f79c758868 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:19:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-47091 namespace: kserve-ci-e2e-test resourceVersion: "14319" uid: fb6496d8-85de-4032-b805-1b2afc7abd6b kind: Event lastTimestamp: "2026-04-16T18:19:22Z" message: InferenceGraph [switch-graph-47091] is Ready metadata: creationTimestamp: "2026-04-16T18:19:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:19:22Z" name: switch-graph-47091.18a6e93f8ff710d0 namespace: kserve-ci-e2e-test resourceVersion: "14321" uid: 6b20005d-1575-4e93-b4ca-6b28dd7310ff reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:41:42.819760Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dbc56-7657877866-wv6tx namespace: kserve-ci-e2e-test resourceVersion: "25609" uid: 3fcbb6aa-4eba-4397-80a4-53930b788235 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-dbc56-7657877866-wv6tx to ip-10-0-128-95.ec2.internal metadata: creationTimestamp: "2026-04-16T18:41:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:41:42Z" name: switch-graph-dbc56-7657877866-wv6tx.18a6ea77b00b2442 namespace: kserve-ci-e2e-test resourceVersion: "25614" uid: 875c61f9-627d-4975-a325-2ce3b18e25c6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d95d69cdf-4h6hm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:43Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dbc56-7657877866-wv6tx namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: 3fcbb6aa-4eba-4397-80a4-53930b788235 kind: Event lastTimestamp: "2026-04-16T18:41:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-dbc56-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:43Z" name: switch-graph-dbc56-7657877866-wv6tx.18a6ea77c0d5b318 namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: 4b73aede-7ee1-4e6d-b191-18ef319b6c4e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:43Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dbc56-7657877866-wv6tx namespace: kserve-ci-e2e-test resourceVersion: "25618" uid: 3fcbb6aa-4eba-4397-80a4-53930b788235 kind: Event lastTimestamp: "2026-04-16T18:41:43Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:41:43Z" name: switch-graph-dbc56-7657877866-wv6tx.18a6ea77ed92d61f namespace: kserve-ci-e2e-test resourceVersion: "25633" uid: 59f3b70f-d459-41ca-b1ac-1394cf2e0719 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dbc56} kind: Pod name: switch-graph-dbc56-7657877866-wv6tx namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: 3fcbb6aa-4eba-4397-80a4-53930b788235 kind: Event lastTimestamp: "2026-04-16T18:41:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c8374c6594fd45817756e1438b782ad7ed814e3785c4fd80a8d4fb659af36654" already present on machine metadata: creationTimestamp: "2026-04-16T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:43Z" name: switch-graph-dbc56-7657877866-wv6tx.18a6ea77eee340cf namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 17b0779e-33ca-49aa-902d-6b321aaa9717 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dbc56} kind: Pod name: switch-graph-dbc56-7657877866-wv6tx namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: 3fcbb6aa-4eba-4397-80a4-53930b788235 kind: Event lastTimestamp: "2026-04-16T18:41:43Z" message: 'Created container: switch-graph-dbc56' metadata: creationTimestamp: "2026-04-16T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:43Z" name: switch-graph-dbc56-7657877866-wv6tx.18a6ea77f1ea060c namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: 9935143e-a118-4557-9ac6-961ac48740e0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dbc56} kind: Pod name: switch-graph-dbc56-7657877866-wv6tx namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: 3fcbb6aa-4eba-4397-80a4-53930b788235 kind: Event lastTimestamp: "2026-04-16T18:41:43Z" message: Started container switch-graph-dbc56 metadata: creationTimestamp: "2026-04-16T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:41:43Z" name: switch-graph-dbc56-7657877866-wv6tx.18a6ea77f264dee0 namespace: kserve-ci-e2e-test resourceVersion: "25640" uid: d3305f6a-c26a-4285-9c4a-9e8e695be1ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dbc56} kind: Pod name: switch-graph-dbc56-7657877866-wv6tx namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: 3fcbb6aa-4eba-4397-80a4-53930b788235 kind: Event lastTimestamp: "2026-04-16T18:58:02Z" message: Stopping container switch-graph-dbc56 metadata: creationTimestamp: "2026-04-16T18:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:58:02Z" name: switch-graph-dbc56-7657877866-wv6tx.18a6eb5bbdda67db namespace: kserve-ci-e2e-test resourceVersion: "32791" uid: 07c6b269-c09f-44f1-9347-415668238739 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:58:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dbc56} kind: Pod name: switch-graph-dbc56-7657877866-wv6tx namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: 3fcbb6aa-4eba-4397-80a4-53930b788235 kind: Event lastTimestamp: "2026-04-16T18:58:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:58:25Z" name: switch-graph-dbc56-7657877866-wv6tx.18a6eb5c8a269c03 namespace: kserve-ci-e2e-test resourceVersion: "33027" uid: 4d41be8f-dac6-4bde-b83e-03c8fce89c6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-95.ec2.internal source: component: kubelet host: ip-10-0-128-95.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-dbc56-7657877866 namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: 95da08c7-0c38-45de-a1df-b1eb337a3e30 kind: Event lastTimestamp: "2026-04-16T18:41:42Z" message: 'Created pod: switch-graph-dbc56-7657877866-wv6tx' metadata: creationTimestamp: "2026-04-16T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:42Z" name: switch-graph-dbc56-7657877866.18a6ea77af9abc96 namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: 267ea185-5ec5-4f3d-ada4-b5f1dd3ebc54 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "25604" uid: 0f4af40f-51e3-498a-bf3e-694c0fa9e77f kind: Event lastTimestamp: "2026-04-16T18:41:42Z" message: Scaled up replica set switch-graph-dbc56-7657877866 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:41:42Z" name: switch-graph-dbc56.18a6ea77aed57b29 namespace: kserve-ci-e2e-test resourceVersion: "25607" uid: 40c111b4-2f7a-4a9d-84ff-6652e1771609 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:41:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "25620" uid: 2b771dd0-396c-448f-a6ac-f4561d1e3eb0 kind: Event lastTimestamp: "2026-04-16T18:41:42Z" message: InferenceGraph [switch-graph-dbc56] is Ready metadata: creationTimestamp: "2026-04-16T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:41:42Z" name: switch-graph-dbc56.18a6ea77b418288a namespace: kserve-ci-e2e-test resourceVersion: "25622" uid: caae566a-5b52-46de-a28c-b4f9f7c794f4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:41:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: c592e304-54d4-4390-9209-a11b1fe12e7d kind: Event lastTimestamp: "2026-04-16T18:42:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:12Z" name: switch-graph-dbc56.18a6ea7b2e7b4c17 namespace: kserve-ci-e2e-test resourceVersion: "25883" uid: 5438235c-3dba-4cb9-9712-4da0e8dfc6a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:41:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dbc56 namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: c592e304-54d4-4390-9209-a11b1fe12e7d kind: Event lastTimestamp: "2026-04-16T18:42:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:42:12Z" name: switch-graph-dbc56.18a6ea7b2f3e73d0 namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: 139d2111-1644-44ff-8f3f-b7b4ed8932be reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33027"