--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:47:20.639274Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-a9fd5-7b8774d5b4-6s425 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: 26969180-d212-46ae-aa32-fe5bf4e0cd40 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-a9fd5-7b8774d5b4-6s425 to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:47:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:47:20Z" name: ensemble-graph-a9fd5-7b8774d5b4-6s425.18a9693d7449c35c namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 23178ac2-0ad7-46c4-88b1-88c3f89e2fdb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:47:20Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-a9fd5-7b8774d5b4-6s425 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 26969180-d212-46ae-aa32-fe5bf4e0cd40 kind: Event lastTimestamp: "2026-04-24T21:47:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-a9fd5-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:21Z" name: ensemble-graph-a9fd5-7b8774d5b4-6s425.18a9693d84542f35 namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: 1e8e5fb4-fa22-4b23-b5f5-667ac5e70dfa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:22Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-a9fd5-7b8774d5b4-6s425 namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 26969180-d212-46ae-aa32-fe5bf4e0cd40 kind: Event lastTimestamp: "2026-04-24T21:47:22Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:47:22Z" name: ensemble-graph-a9fd5-7b8774d5b4-6s425.18a9693de7183dc8 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 967e9b48-fc7e-4589-a3fb-c58ad5de199f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9fd5} kind: Pod name: ensemble-graph-a9fd5-7b8774d5b4-6s425 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 26969180-d212-46ae-aa32-fe5bf4e0cd40 kind: Event lastTimestamp: "2026-04-24T21:47:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:22Z" name: ensemble-graph-a9fd5-7b8774d5b4-6s425.18a9693de85ddd0a namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 4f517723-3eef-4598-b193-2d84ca80395c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9fd5} kind: Pod name: ensemble-graph-a9fd5-7b8774d5b4-6s425 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 26969180-d212-46ae-aa32-fe5bf4e0cd40 kind: Event lastTimestamp: "2026-04-24T21:47:22Z" message: 'Created container: ensemble-graph-a9fd5' metadata: creationTimestamp: "2026-04-24T21:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:22Z" name: ensemble-graph-a9fd5-7b8774d5b4-6s425.18a9693deb41ac3a namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: 3a091724-95d8-4a94-b286-cdc590ff55ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9fd5} kind: Pod name: ensemble-graph-a9fd5-7b8774d5b4-6s425 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 26969180-d212-46ae-aa32-fe5bf4e0cd40 kind: Event lastTimestamp: "2026-04-24T21:47:22Z" message: Started container ensemble-graph-a9fd5 metadata: creationTimestamp: "2026-04-24T21:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:22Z" name: ensemble-graph-a9fd5-7b8774d5b4-6s425.18a9693debbc8d04 namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 9e957c68-f878-4a38-bae2-0a66a8bc02ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9fd5} kind: Pod name: ensemble-graph-a9fd5-7b8774d5b4-6s425 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 26969180-d212-46ae-aa32-fe5bf4e0cd40 kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: Stopping container ensemble-graph-a9fd5 metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:35Z" name: ensemble-graph-a9fd5-7b8774d5b4-6s425.18a969b0a71ccb59 namespace: kserve-ci-e2e-test resourceVersion: "23706" uid: 81098bf6-d9dd-475a-b03b-f699913d5253 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9fd5} kind: Pod name: ensemble-graph-a9fd5-7b8774d5b4-6s425 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 26969180-d212-46ae-aa32-fe5bf4e0cd40 kind: Event lastTimestamp: "2026-04-24T21:56:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:04Z" name: ensemble-graph-a9fd5-7b8774d5b4-6s425.18a969b189e1437b namespace: kserve-ci-e2e-test resourceVersion: "24046" uid: 38b34f3c-dcc3-4b53-b4dc-bbbf803b260c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-a9fd5-7b8774d5b4 namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: deaf5352-e1b2-4178-b068-78ff7d188974 kind: Event lastTimestamp: "2026-04-24T21:47:20Z" message: 'Created pod: ensemble-graph-a9fd5-7b8774d5b4-6s425' metadata: creationTimestamp: "2026-04-24T21:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:20Z" name: ensemble-graph-a9fd5-7b8774d5b4.18a9693d73e6e324 namespace: kserve-ci-e2e-test resourceVersion: "20180" uid: cde5487d-5038-4f83-84ae-8a4c2b1afdf5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: 292d58e3-8f5f-4926-aa3d-4acb52c3a014 kind: Event lastTimestamp: "2026-04-24T21:47:20Z" message: Scaled up replica set ensemble-graph-a9fd5-7b8774d5b4 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:20Z" name: ensemble-graph-a9fd5.18a9693d733335a6 namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: a6990747-1610-435d-88ef-6762573aee76 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 333354ef-4f66-4d47-aa3e-b5f13e63477b kind: Event lastTimestamp: "2026-04-24T21:47:20Z" message: InferenceGraph [ensemble-graph-a9fd5] is Ready metadata: creationTimestamp: "2026-04-24T21:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:20Z" name: ensemble-graph-a9fd5.18a9693d76a47d20 namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: 4e9a058a-9bb1-47f9-b14f-931f31e7f200 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:47:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: abd576a9-7cf7-4ce4-ad21-8b82332bcbe2 kind: Event lastTimestamp: "2026-04-24T21:48:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:05Z" name: ensemble-graph-a9fd5.18a96940f1de1102 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 178d4cc9-5b76-4842-a4cd-21469b58da9a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:47:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: abd576a9-7cf7-4ce4-ad21-8b82332bcbe2 kind: Event lastTimestamp: "2026-04-24T21:48:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:05Z" name: ensemble-graph-a9fd5.18a96940f21f0f0e namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: 5a95f9e4-c2b5-4324-ae69-5a6fff737a9b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:46:10.384535Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc namespace: kserve-ci-e2e-test resourceVersion: "19204" uid: 40bf3ac8-34f0-4165-8262-5138231ed11f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:46:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:46:10Z" name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc.18a9692d18c78589 namespace: kserve-ci-e2e-test resourceVersion: "19208" uid: 1c0c8b28-9938-4a10-a673-37c4c888259b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:10Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: 40bf3ac8-34f0-4165-8262-5138231ed11f kind: Event lastTimestamp: "2026-04-24T21:46:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-dd3fc-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:10Z" name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc.18a9692d280e503b namespace: kserve-ci-e2e-test resourceVersion: "19222" uid: 7ff0be17-f260-4909-b872-6a804695bc19 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:11Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc namespace: kserve-ci-e2e-test resourceVersion: "19213" uid: 40bf3ac8-34f0-4165-8262-5138231ed11f kind: Event lastTimestamp: "2026-04-24T21:46:11Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:46:11Z" name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc.18a9692d56060dcd namespace: kserve-ci-e2e-test resourceVersion: "19230" uid: 60ad82ca-5c1d-4b9a-b3b8-5b4d06d2ec34 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dd3fc} kind: Pod name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: 40bf3ac8-34f0-4165-8262-5138231ed11f kind: Event lastTimestamp: "2026-04-24T21:46:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:11Z" name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc.18a9692d57654730 namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: 7dae0e8a-cbbd-48bb-9c0d-e19fca98227b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dd3fc} kind: Pod name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: 40bf3ac8-34f0-4165-8262-5138231ed11f kind: Event lastTimestamp: "2026-04-24T21:46:11Z" message: 'Created container: ensemble-graph-dd3fc' metadata: creationTimestamp: "2026-04-24T21:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:11Z" name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc.18a9692d5a49a50e namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: d6f3e5a1-5652-4530-8fc2-779895577c63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dd3fc} kind: Pod name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: 40bf3ac8-34f0-4165-8262-5138231ed11f kind: Event lastTimestamp: "2026-04-24T21:46:11Z" message: Started container ensemble-graph-dd3fc metadata: creationTimestamp: "2026-04-24T21:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:11Z" name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc.18a9692d5ac03ccd namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: 79c696a0-c5b2-4b0f-bcb7-c72915c2ac54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dd3fc} kind: Pod name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: 40bf3ac8-34f0-4165-8262-5138231ed11f kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: Stopping container ensemble-graph-dd3fc metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:20Z" name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc.18a9692f70b586bb namespace: kserve-ci-e2e-test resourceVersion: "19328" uid: 1bef4ed0-7949-40ec-8d82-0d249fe01c53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dd3fc} kind: Pod name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: 40bf3ac8-34f0-4165-8262-5138231ed11f kind: Event lastTimestamp: "2026-04-24T21:46:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:48Z" name: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc.18a9693008c4e9dd namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 6dc53b84-add3-4619-bf41-421240d63fd6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-dd3fc-6bb9dd7f4b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: f9ad4e6c-f7b0-476e-afd1-6b1e26b71533 kind: Event lastTimestamp: "2026-04-24T21:46:10Z" message: 'Created pod: ensemble-graph-dd3fc-6bb9dd7f4b-fj4mc' metadata: creationTimestamp: "2026-04-24T21:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:10Z" name: ensemble-graph-dd3fc-6bb9dd7f4b.18a9692d185d5721 namespace: kserve-ci-e2e-test resourceVersion: "19205" uid: da140eb0-20d5-4aad-812e-379635798688 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "19197" uid: 244e27f1-5213-405e-a0cb-6ebc8b0e66a8 kind: Event lastTimestamp: "2026-04-24T21:46:10Z" message: Scaled up replica set ensemble-graph-dd3fc-6bb9dd7f4b from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:10Z" name: ensemble-graph-dd3fc.18a9692d17c8570b namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 93187237-3bad-40fb-b7b7-556b77587337 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "19215" uid: 72481d0e-dc17-45ce-a927-4256c24414c2 kind: Event lastTimestamp: "2026-04-24T21:46:10Z" message: InferenceGraph [ensemble-graph-dd3fc] is Ready metadata: creationTimestamp: "2026-04-24T21:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:10Z" name: ensemble-graph-dd3fc.18a9692d1d2d4a6f namespace: kserve-ci-e2e-test resourceVersion: "19217" uid: cfe84f52-7e41-44a9-aa85-a4317cfdd08a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:36:35.577025Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-54155-predictor-db578f77d-7fmtl to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:36:35Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a968a74394f521 namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: e5c0c85b-355c-43d9-81a6-2285074235a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:36:35Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a968a75c69acba namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: cef695e0-d235-4b96-adaf-114bdbdfaa18 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a968a75ddf1300 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 0a605a14-afba-4c50-9aaa-0366a707a421 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a968a760d7bb61 namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 57f0b29e-92da-436e-a7d9-ed81e6f825ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a968a7614736d5 namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: dfc9c638-0b0b-47b9-b326-9fb7b2a15913 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a968a76151fe07 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 8952bd34-78fe-412a-abe3-28f9177ec5d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a968a7643f4f50 namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: d390f4ca-5626-416a-b3d1-3b68b6e77b09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a968a764afc6e2 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: 6cf022da-b262-410a-95ab-5997eb41de30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:45:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:53Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a968a7a7edbb9d namespace: kserve-ci-e2e-test resourceVersion: "19022" uid: 270ecd6a-3d90-4eb8-89c2-a8f373f6623f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a9692861c1872f namespace: kserve-ci-e2e-test resourceVersion: "18896" uid: 463fd372-d92c-4210-8559-3f01a95ecc3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a9692861c229d7 namespace: kserve-ci-e2e-test resourceVersion: "18899" uid: 981782e3-23a3-4d30-823f-112cc23576fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-54155-predictor-db578f77d-7fmtl namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: affb13ef-1273-40ee-a013-bddd85341559 kind: Event lastTimestamp: "2026-04-24T21:45:53Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:53Z" name: error-404-isvc-54155-predictor-db578f77d-7fmtl.18a969291c3fe8f1 namespace: kserve-ci-e2e-test resourceVersion: "19019" uid: 4babf8a6-7e70-42f0-9729-51a4a284b871 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-54155-predictor-db578f77d namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: b18061cc-a584-48fd-a684-b2c68682e33c kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: 'Created pod: error-404-isvc-54155-predictor-db578f77d-7fmtl' metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:35Z" name: error-404-isvc-54155-predictor-db578f77d.18a968a742806c6c namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: 394f76b8-e135-43fa-bc02-a814ec81e965 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 5522111d-c5b3-44b5-aa54-eb8bfabc455e kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Scaled up replica set error-404-isvc-54155-predictor-db578f77d from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:35Z" name: error-404-isvc-54155-predictor.18a968a73f2ed354 namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 05417795-5d01-4964-8379-72c996ddf5ad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:36:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: f99faba4-a0ab-4ea6-b2a9-f31213999720 kind: Event lastTimestamp: "2026-04-24T21:37:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:20Z" name: error-404-isvc-54155-predictor.18a968aabdc32b3b namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: 51fb687e-ea07-48d4-a3f7-6a153ba76d20 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:36:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: f99faba4-a0ab-4ea6-b2a9-f31213999720 kind: Event lastTimestamp: "2026-04-24T21:37:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:20Z" name: error-404-isvc-54155-predictor.18a968aabe29ad41 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: fdc76be1-914c-4eb8-87f9-15a3486c508a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: f99faba4-a0ab-4ea6-b2a9-f31213999720 kind: Event lastTimestamp: "2026-04-24T21:38:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:05Z" name: error-404-isvc-54155-predictor.18a968b53b4ce2b1 namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: 5e003f0a-ab20-4e4c-b232-34ebae72d23d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: f99faba4-a0ab-4ea6-b2a9-f31213999720 kind: Event lastTimestamp: "2026-04-24T21:38:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:05Z" name: error-404-isvc-54155-predictor.18a968b53b9e2439 namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: cef7aafd-3fc7-4ee2-a9d5-d41fb6a13548 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54155 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 2e7fc742-d254-4308-9ad5-22a0eeffbe5a kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:23Z" name: error-404-isvc-54155.18a968a73f13b7ed namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: f07fe7ee-cf70-4a3d-9e6c-34bf2ff43e45 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54155 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 2e7fc742-d254-4308-9ad5-22a0eeffbe5a kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-54155": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-54155": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:35Z" name: error-404-isvc-54155.18a968a73f886132 namespace: kserve-ci-e2e-test resourceVersion: "14637" uid: c5d3ce9c-ce98-4470-8ff2-419af43df306 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54155 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 2e7fc742-d254-4308-9ad5-22a0eeffbe5a kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-54155": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:35Z" name: error-404-isvc-54155.18a968a73f88bff6 namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 57f737a2-10d2-4757-99f1-f2ee129be4d7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54155 namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 2e7fc742-d254-4308-9ad5-22a0eeffbe5a kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: InferenceService [error-404-isvc-54155] is Ready metadata: creationTimestamp: "2026-04-24T21:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:23Z" name: error-404-isvc-54155.18a968b265bae6e5 namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 363614f3-1d66-48e9-b0d8-1f54c6197701 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:45:50.308550Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18956" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6445b-predictor-566b89cd56-67vrw to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a969286c284be9 namespace: kserve-ci-e2e-test resourceVersion: "18962" uid: 59742609-0c7e-4f10-9d29-7848dae283eb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18968" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a9692885bad693 namespace: kserve-ci-e2e-test resourceVersion: "18977" uid: 729295d1-274d-4c3e-8be9-2af996037561 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a96928870ee343 namespace: kserve-ci-e2e-test resourceVersion: "18979" uid: 31776754-4f57-45f7-8add-32038ea9bb5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a969288aac07a0 namespace: kserve-ci-e2e-test resourceVersion: "18980" uid: 0f6b4775-ac19-4ad8-895e-ce4b9b7d4263 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a969288b1f8bea namespace: kserve-ci-e2e-test resourceVersion: "18981" uid: c452fd29-b717-4a6f-88cf-9a31ada40d12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a969288b2a7e24 namespace: kserve-ci-e2e-test resourceVersion: "18982" uid: 4868f470-bc48-4cc6-93e9-35c6a038de52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a969288dc3a9b3 namespace: kserve-ci-e2e-test resourceVersion: "18983" uid: d86a7802-d6c1-46fe-9dbd-fa3e4be6922f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a969288e30b644 namespace: kserve-ci-e2e-test resourceVersion: "18984" uid: 00f37157-bc35-4cf5-98d7-4ea68ec2e2b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:46:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:26Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a969288f08fcf1 namespace: kserve-ci-e2e-test resourceVersion: "19506" uid: 3f57b9b9-fd2c-432b-b5be-416027fc203e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a96938c6fbe8c5 namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: a8f9ba8d-2781-4cce-a045-a3be600a4036 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a96938c6fcfe59 namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 8fe3622f-eaf3-496a-9f30-8179f5a4433f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6445b-predictor-566b89cd56-67vrw namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 0b3e97f0-f7b9-44f1-b0b1-5bbaf899a83c kind: Event lastTimestamp: "2026-04-24T21:47:01Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:01Z" name: error-404-isvc-6445b-predictor-566b89cd56-67vrw.18a969391775ba7b namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: 4ed05ab7-12bf-415d-a010-ea8c79a80bb9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6445b-predictor-566b89cd56 namespace: kserve-ci-e2e-test resourceVersion: "18948" uid: 074bf49b-9bc9-4581-b2c7-75a1cb973239 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: 'Created pod: error-404-isvc-6445b-predictor-566b89cd56-67vrw' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor-566b89cd56.18a969286bc53f35 namespace: kserve-ci-e2e-test resourceVersion: "18958" uid: 779cdb6b-e939-4adf-9ead-91879d5fa3f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18946" uid: b2d5a493-764f-4973-9206-fc3f1c602e5c kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Scaled up replica set error-404-isvc-6445b-predictor-566b89cd56 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b-predictor.18a969286a29f979 namespace: kserve-ci-e2e-test resourceVersion: "18950" uid: e2560348-8f54-4702-afbd-f4efce2c35d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: c619081c-8569-4974-8bb5-80a334532c56 kind: Event lastTimestamp: "2026-04-24T21:46:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:35Z" name: error-404-isvc-6445b-predictor.18a9692be8b7b40b namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: a6d628ba-db6b-43ad-a7d6-bae74d2ac1f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: c619081c-8569-4974-8bb5-80a334532c56 kind: Event lastTimestamp: "2026-04-24T21:46:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:35Z" name: error-404-isvc-6445b-predictor.18a9692be9230f07 namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 43b6e904-a1ac-48e5-8a4e-15ee1dc11495 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19145" uid: c619081c-8569-4974-8bb5-80a334532c56 kind: Event lastTimestamp: "2026-04-24T21:46:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:50Z" name: error-404-isvc-6445b-predictor.18a9692f67dc5aa6 namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: c0d784f9-d274-4f15-9928-c1a46b370ab1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19145" uid: c619081c-8569-4974-8bb5-80a334532c56 kind: Event lastTimestamp: "2026-04-24T21:46:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:50Z" name: error-404-isvc-6445b-predictor.18a9692f684a23dc namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 3ff5670f-720f-48a7-a0aa-66611b4a076e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6445b namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: 960acc5d-9c96-4e7d-ad43-05f89d260aa6 kind: Event lastTimestamp: "2026-04-24T21:46:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:37Z" name: error-404-isvc-6445b.18a969286a08b837 namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: f26d21f4-d85c-4b60-9b2c-a274d6965aac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6445b namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: 960acc5d-9c96-4e7d-ad43-05f89d260aa6 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-6445b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6445b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b.18a969286a5912e7 namespace: kserve-ci-e2e-test resourceVersion: "18952" uid: eba4b7b7-2f6e-47f0-b887-1e9d8b8fa192 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6445b namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: 960acc5d-9c96-4e7d-ad43-05f89d260aa6 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6445b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:50Z" name: error-404-isvc-6445b.18a969286a59606b namespace: kserve-ci-e2e-test resourceVersion: "18953" uid: 7937902b-0eb4-4a74-be0a-42e95486bf03 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6445b namespace: kserve-ci-e2e-test resourceVersion: "19606" uid: 960acc5d-9c96-4e7d-ad43-05f89d260aa6 kind: Event lastTimestamp: "2026-04-24T21:46:36Z" message: InferenceService [error-404-isvc-6445b] is Ready metadata: creationTimestamp: "2026-04-24T21:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:36Z" name: error-404-isvc-6445b.18a969334a7ab434 namespace: kserve-ci-e2e-test resourceVersion: "19607" uid: beb87103-0692-44a4-b421-f36f6928365d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:55:35.699997Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23783" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6842b-predictor-565dd576f8-rwz65 to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:55:35Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969b0b834ebef namespace: kserve-ci-e2e-test resourceVersion: "23787" uid: fd2bb587-5a11-4f87-83a9-35fe1aca329d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:55:36Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969b0d1823146 namespace: kserve-ci-e2e-test resourceVersion: "23804" uid: 2b8cc51d-1c98-4122-937e-6c201feea976 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969b0d2d16032 namespace: kserve-ci-e2e-test resourceVersion: "23807" uid: d368a46b-f122-4e2f-bad2-34d8600f823c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969b0d5addc8b namespace: kserve-ci-e2e-test resourceVersion: "23813" uid: ec0859ae-00f1-4912-97ae-41373aae8edd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969b0d61d9723 namespace: kserve-ci-e2e-test resourceVersion: "23815" uid: 3bbb3438-2df6-46d2-96fd-f4cb50f3989e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969b0d62852a1 namespace: kserve-ci-e2e-test resourceVersion: "23816" uid: 6c2b3786-e786-43dc-b0ce-be8456727b1f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969b0d930283d namespace: kserve-ci-e2e-test resourceVersion: "23817" uid: 4541f521-589e-4ede-99f1-39a6573a1bda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969b0d9aecf5b namespace: kserve-ci-e2e-test resourceVersion: "23818" uid: 76e33d45-6334-4bc6-8cf9-a7f2f5a56ba2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:55:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:56:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:55:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:13Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969b12e2abd0d namespace: kserve-ci-e2e-test resourceVersion: "24123" uid: 4418f3af-de25-4df0-a5d3-6a4ef7c89505 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:45Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969c109744c1c namespace: kserve-ci-e2e-test resourceVersion: "24614" uid: 54981268-cee2-4760-a7d7-54c5d354a797 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:45Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969c10977c9ae namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 3e70e375-8a7b-4f98-954f-98a657a7dbda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6842b-predictor-565dd576f8-rwz65 namespace: kserve-ci-e2e-test resourceVersion: "23785" uid: 61ed92de-cc0f-4baf-9b8d-8bb6e4019d55 kind: Event lastTimestamp: "2026-04-24T21:56:48Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:48Z" name: error-404-isvc-6842b-predictor-565dd576f8-rwz65.18a969c1b6796d3f namespace: kserve-ci-e2e-test resourceVersion: "24738" uid: 19c98138-ba3f-4cbd-a6de-40326ac247cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6842b-predictor-565dd576f8 namespace: kserve-ci-e2e-test resourceVersion: "23776" uid: 324edc6b-1a3c-42b5-a087-34721e979eb1 kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: 'Created pod: error-404-isvc-6842b-predictor-565dd576f8-rwz65' metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:55:35Z" name: error-404-isvc-6842b-predictor-565dd576f8.18a969b0b7c44928 namespace: kserve-ci-e2e-test resourceVersion: "23784" uid: 6299305c-4914-4e96-b6c6-f663b476d3ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23772" uid: 6d5ebad0-7c87-4863-bdc0-28c587e891bf kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: Scaled up replica set error-404-isvc-6842b-predictor-565dd576f8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:55:35Z" name: error-404-isvc-6842b-predictor.18a969b0b5e963d3 namespace: kserve-ci-e2e-test resourceVersion: "23778" uid: 1a6b2353-f7bf-44aa-a283-65e4be0339e6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23775" uid: 37ab0fd7-4680-42ae-8ab2-9c54be7e9739 kind: Event lastTimestamp: "2026-04-24T21:55:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:55:50Z" name: error-404-isvc-6842b-predictor.18a969b434e2cade namespace: kserve-ci-e2e-test resourceVersion: "23948" uid: d1dc5b40-354a-47f1-9edb-a9e0bd6ea450 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23775" uid: 37ab0fd7-4680-42ae-8ab2-9c54be7e9739 kind: Event lastTimestamp: "2026-04-24T21:55:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:55:50Z" name: error-404-isvc-6842b-predictor.18a969b435941c90 namespace: kserve-ci-e2e-test resourceVersion: "23950" uid: a14ced38-2c81-456c-8e96-f1fb019d5c5a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:56:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23947" uid: 37ab0fd7-4680-42ae-8ab2-9c54be7e9739 kind: Event lastTimestamp: "2026-04-24T21:56:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:35Z" name: error-404-isvc-6842b-predictor.18a969b7b4ec9eac namespace: kserve-ci-e2e-test resourceVersion: "24489" uid: f2af5b66-22a5-4af9-bc16-c7c4b025af79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:56:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23947" uid: 37ab0fd7-4680-42ae-8ab2-9c54be7e9739 kind: Event lastTimestamp: "2026-04-24T21:56:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:35Z" name: error-404-isvc-6842b-predictor.18a969b7b5a339db namespace: kserve-ci-e2e-test resourceVersion: "24490" uid: bd6f31b5-40f9-4c71-b4c3-5a7b847296c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6842b namespace: kserve-ci-e2e-test resourceVersion: "23743" uid: 8d13b5d3-6bd2-4b93-bf0c-562ec94a3614 kind: Event lastTimestamp: "2026-04-24T21:56:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:23Z" name: error-404-isvc-6842b.18a969b0b5b590ca namespace: kserve-ci-e2e-test resourceVersion: "24361" uid: 20713253-412e-4f48-8f22-56f8405a1e1d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6842b namespace: kserve-ci-e2e-test resourceVersion: "23743" uid: 8d13b5d3-6bd2-4b93-bf0c-562ec94a3614 kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-6842b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6842b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:55:35Z" name: error-404-isvc-6842b.18a969b0b62001b3 namespace: kserve-ci-e2e-test resourceVersion: "23780" uid: 743a8aa2-79f3-4d87-bb9e-398b0dd02cad reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6842b namespace: kserve-ci-e2e-test resourceVersion: "23743" uid: 8d13b5d3-6bd2-4b93-bf0c-562ec94a3614 kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6842b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:55:35Z" name: error-404-isvc-6842b.18a969b0b6206894 namespace: kserve-ci-e2e-test resourceVersion: "23781" uid: 65dde8c5-81e3-4c66-b5d7-c64dff2d85e8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6842b namespace: kserve-ci-e2e-test resourceVersion: "24359" uid: 8d13b5d3-6bd2-4b93-bf0c-562ec94a3614 kind: Event lastTimestamp: "2026-04-24T21:56:23Z" message: InferenceService [error-404-isvc-6842b] is Ready metadata: creationTimestamp: "2026-04-24T21:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:23Z" name: error-404-isvc-6842b.18a969bbe9d513ab namespace: kserve-ci-e2e-test resourceVersion: "24360" uid: 5536440d-5ffb-4ef6-bd1e-f0300156d2a8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:56:45.980981Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24674" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6941b-predictor-db66cc574-zc9wx to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:56:45Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c11547ba39 namespace: kserve-ci-e2e-test resourceVersion: "24678" uid: 6a27a545-db38-4ed2-92c7-8b643b05515f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T21:56:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6941b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:46Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c1232abf57 namespace: kserve-ci-e2e-test resourceVersion: "24689" uid: 52a6f6d7-ee42-4fff-a13c-a42b02cc3f26 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24683" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:56:47Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c1528d2423 namespace: kserve-ci-e2e-test resourceVersion: "24707" uid: 86195460-2ba2-40be-bb29-75aceb2f5d88 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c1547bbbab namespace: kserve-ci-e2e-test resourceVersion: "24710" uid: 6b14542c-ed17-47e1-9537-f0079db05a07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c156c54883 namespace: kserve-ci-e2e-test resourceVersion: "24714" uid: 2344aeac-55d8-4e73-8b45-da88f4f9b556 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c15735712a namespace: kserve-ci-e2e-test resourceVersion: "24715" uid: 47ab8b9e-8e47-4e13-9e37-7343b20cf89e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c157409196 namespace: kserve-ci-e2e-test resourceVersion: "24716" uid: e23fe537-1cfb-4737-a7c3-b6da12f55fb3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c15a31afd5 namespace: kserve-ci-e2e-test resourceVersion: "24717" uid: 3ce436f1-15bd-4c12-a26d-900f3970ab6a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c15aa4ce7e namespace: kserve-ci-e2e-test resourceVersion: "24718" uid: 31073954-79d0-4726-a3e7-86c15e37eb73 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T21:57:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:24Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a969c1c6249b28 namespace: kserve-ci-e2e-test resourceVersion: "25097" uid: 5b8004dc-fbe5-4331-b2e2-e00f0f99ea7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T22:06:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:06:00Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a96a4237aafeb8 namespace: kserve-ci-e2e-test resourceVersion: "29055" uid: b513d8ca-0ade-4477-bf2f-8957bdcc1c4b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6941b-predictor-db66cc574-zc9wx namespace: kserve-ci-e2e-test resourceVersion: "24676" uid: 6719c2fc-f689-4854-9eb7-9dbba8fe4bfb kind: Event lastTimestamp: "2026-04-24T22:06:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:06:00Z" name: error-404-isvc-6941b-predictor-db66cc574-zc9wx.18a96a4237ab8c88 namespace: kserve-ci-e2e-test resourceVersion: "29056" uid: c9d9fcfb-71d3-4cfa-81ff-dc37adc8825b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6941b-predictor-db66cc574 namespace: kserve-ci-e2e-test resourceVersion: "24668" uid: 8c92e17b-34ed-4992-8619-9a3a36d497f9 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: 'Created pod: error-404-isvc-6941b-predictor-db66cc574-zc9wx' metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:45Z" name: error-404-isvc-6941b-predictor-db66cc574.18a969c114dd450d namespace: kserve-ci-e2e-test resourceVersion: "24675" uid: 6011da14-dec4-40e5-925d-d9bbac689c83 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24666" uid: 65f0edb7-feb0-4b6e-870e-947185f7085c kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: Scaled up replica set error-404-isvc-6941b-predictor-db66cc574 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:45Z" name: error-404-isvc-6941b-predictor.18a969c112b43ea0 namespace: kserve-ci-e2e-test resourceVersion: "24670" uid: 68c6536d-82f8-4394-aba8-dcbed481f383 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:57:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24667" uid: 172d210d-258a-4ca4-973a-f9c668e779b8 kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:57:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:15Z" name: error-404-isvc-6941b-predictor.18a969c4915a9b51 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 6e94adfa-33ae-4a66-b56f-b859a0c269f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:57:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24667" uid: 172d210d-258a-4ca4-973a-f9c668e779b8 kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:57:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:15Z" name: error-404-isvc-6941b-predictor.18a969c491a4f568 namespace: kserve-ci-e2e-test resourceVersion: "25023" uid: c9094ebb-f5b2-4042-bd19-77e3555a54f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24856" uid: 172d210d-258a-4ca4-973a-f9c668e779b8 kind: Event lastTimestamp: "2026-04-24T21:58:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:01Z" name: error-404-isvc-6941b-predictor.18a969cb8f708837 namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: d7fc09f3-57a5-44b2-b055-6e7fa923cbd6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24856" uid: 172d210d-258a-4ca4-973a-f9c668e779b8 kind: Event lastTimestamp: "2026-04-24T21:58:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:01Z" name: error-404-isvc-6941b-predictor.18a969cb901877ed namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: c912dfe2-0ba5-4c68-8f0d-592ea467af1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "24647" uid: a8da894f-576b-444e-ace2-dc4c9b66e70c kind: Event lastTimestamp: "2026-04-24T21:57:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:35Z" name: error-404-isvc-6941b.18a969c1129076ee namespace: kserve-ci-e2e-test resourceVersion: "25194" uid: b4f98cc8-b60d-4ab5-9873-2b7845849348 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "25190" uid: a8da894f-576b-444e-ace2-dc4c9b66e70c kind: Event lastTimestamp: "2026-04-24T21:57:35Z" message: InferenceService [error-404-isvc-6941b] is Ready metadata: creationTimestamp: "2026-04-24T21:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:35Z" name: error-404-isvc-6941b.18a969cc85d6f562 namespace: kserve-ci-e2e-test resourceVersion: "25191" uid: f7f0ed2c-69c5-4839-9d47-09cb1727f214 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:35:55.527920Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14088" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:35:55Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9689df0782092 namespace: kserve-ci-e2e-test resourceVersion: "14092" uid: e6c24777-b4e5-4191-bc74-a656315e486d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14098" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:35:55Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9689e09ab4185 namespace: kserve-ci-e2e-test resourceVersion: "14112" uid: 8aaaa3d1-d067-4fd8-98d8-49e5a779a351 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9689e0b1cb8d8 namespace: kserve-ci-e2e-test resourceVersion: "14115" uid: 4bc9715d-8ab6-4266-9c55-5ea8737e814f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:35:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:56Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9689e0db92b86 namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: fd4b524e-c302-4fef-b133-6d7463b476dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:35:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:56Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9689e0e414427 namespace: kserve-ci-e2e-test resourceVersion: "14118" uid: 3b6e73fd-4655-4762-986d-389bbff63c4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:35:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:56Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9689e0e4d17d1 namespace: kserve-ci-e2e-test resourceVersion: "14119" uid: 6f4c35ed-1db3-43bf-be65-cf6c2c5e188d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:35:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:56Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9689e113a17e9 namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: b1a389b5-801e-42f4-943c-620b01bb7e38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:35:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:56Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9689e11a9e370 namespace: kserve-ci-e2e-test resourceVersion: "14121" uid: 71bf0f02-05b1-4241-af03-fd5d4dd53e56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:45:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:13Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9689e4fe4116c namespace: kserve-ci-e2e-test resourceVersion: "18567" uid: 39ecf544-c152-40e1-b3c9-790dab71df41 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9691f21dcb03d namespace: kserve-ci-e2e-test resourceVersion: "18496" uid: d56b00c1-17b0-4128-a353-c45a17f3885a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9691f21ddb531 namespace: kserve-ci-e2e-test resourceVersion: "18500" uid: a84aac5c-4b87-4c55-a04f-d7b1d66648db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: be31fcea-aebe-4dbd-b1e7-4776f3785a40 kind: Event lastTimestamp: "2026-04-24T21:45:13Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:13Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt.18a9691fc43ca3c8 namespace: kserve-ci-e2e-test resourceVersion: "18565" uid: 4d7f7b2f-8960-40e3-8edd-5c41da4ef065 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6d3ba-predictor-5d779b8c86 namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 9c21e908-325c-46d2-8b50-f8fb2740964b kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'Created pod: error-404-isvc-6d3ba-predictor-5d779b8c86-cqxlt' metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:55Z" name: error-404-isvc-6d3ba-predictor-5d779b8c86.18a9689df033213e namespace: kserve-ci-e2e-test resourceVersion: "14090" uid: da3d9665-3b25-4453-8753-32989999b696 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: fad4b839-bf32-42a3-8c57-0f91b5815dd1 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Scaled up replica set error-404-isvc-6d3ba-predictor-5d779b8c86 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:55Z" name: error-404-isvc-6d3ba-predictor.18a9689dedd44a60 namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: 1126cd3f-bc77-43a9-9c2c-5bffa36e9619 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:36:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: f15e1a28-ca8c-4cce-ad22-697fb58b48a6 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: error-404-isvc-6d3ba-predictor.18a968a16c5a479a namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 036b0182-5e57-441c-b503-10a1868679fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:36:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: f15e1a28-ca8c-4cce-ad22-697fb58b48a6 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: error-404-isvc-6d3ba-predictor.18a968a16cc4fdcf namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: ca5ade0a-0c24-4b50-9926-b8650116a761 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:36:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: f15e1a28-ca8c-4cce-ad22-697fb58b48a6 kind: Event lastTimestamp: "2026-04-24T21:37:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:36:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:10Z" name: error-404-isvc-6d3ba-predictor.18a968a86a55a3cf namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 464f14bb-d528-413e-a55c-342fadca2fa6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:36:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: f15e1a28-ca8c-4cce-ad22-697fb58b48a6 kind: Event lastTimestamp: "2026-04-24T21:37:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:36:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:10Z" name: error-404-isvc-6d3ba-predictor.18a968a86aad398e namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: fa2677c3-5961-4666-8f21-de9eee150a6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: 362cac4e-bfd0-4d99-b7d5-2c78a41ac9c9 kind: Event lastTimestamp: "2026-04-24T21:36:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:43Z" name: error-404-isvc-6d3ba.18a9689deda24fd9 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: f395c4f5-61a8-4ca8-9f4c-5022b96a79c1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: 362cac4e-bfd0-4d99-b7d5-2c78a41ac9c9 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-6d3ba": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6d3ba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:55Z" name: error-404-isvc-6d3ba.18a9689dee3047a6 namespace: kserve-ci-e2e-test resourceVersion: "14085" uid: 1a2f550a-77a0-4d37-9d3b-14a5bc46b927 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: 362cac4e-bfd0-4d99-b7d5-2c78a41ac9c9 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6d3ba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:55Z" name: error-404-isvc-6d3ba.18a9689dee30a923 namespace: kserve-ci-e2e-test resourceVersion: "14086" uid: 934920c9-087c-45b2-8060-243ac05a62a8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 362cac4e-bfd0-4d99-b7d5-2c78a41ac9c9 kind: Event lastTimestamp: "2026-04-24T21:36:43Z" message: InferenceService [error-404-isvc-6d3ba] is Ready metadata: creationTimestamp: "2026-04-24T21:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:43Z" name: error-404-isvc-6d3ba.18a968a90edaa14d namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 394be604-6341-4204-8466-fe7db69f463a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:34:25.506435Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13054" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:34:25Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a96888fac541bb namespace: kserve-ci-e2e-test resourceVersion: "13058" uid: 327f27b2-7b46-48ae-b7e5-eeafa8555330 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13061" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:34:25Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a96889179c1828 namespace: kserve-ci-e2e-test resourceVersion: "13093" uid: dcb1c9d9-59ea-4f96-99d9-358aa5aeb5a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:34:26Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:26Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a968891974528c namespace: kserve-ci-e2e-test resourceVersion: "13095" uid: f0a48ae3-d486-4003-afb8-a39d14e428c1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" in 16.386s (16.386s including waiting). Image size: 1334822504 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9688cea2caaa2 namespace: kserve-ci-e2e-test resourceVersion: "13285" uid: ca79b47a-9e7e-4c75-a6fc-c87001a822ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9688cee059bde namespace: kserve-ci-e2e-test resourceVersion: "13288" uid: ef98d201-b476-4f98-89ca-edc8bcf8c4ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9688cee9329aa namespace: kserve-ci-e2e-test resourceVersion: "13290" uid: 93681b48-7b01-4802-ac62-19c8c80d63ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9688ceec41a28 namespace: kserve-ci-e2e-test resourceVersion: "13291" uid: 663f7d25-98e1-48db-af98-eaa31965a9ca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.416s (2.416s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:44Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9688d7ed0ed3d namespace: kserve-ci-e2e-test resourceVersion: "13318" uid: 26c4dd21-d890-4712-b9e1-efa4393efdc2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:44Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9688d82ab2775 namespace: kserve-ci-e2e-test resourceVersion: "13321" uid: f033ff2d-396a-46b5-b7f3-3df317208e17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:44Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9688d832fc8e9 namespace: kserve-ci-e2e-test resourceVersion: "13323" uid: 06d33626-bcf3-4216-9410-b7c1a4e1fe23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:35:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:22Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9688df623ed54 namespace: kserve-ci-e2e-test resourceVersion: "13671" uid: 3e341431-c231-4f90-90f9-f851af4c9032 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9689de6ecbafd namespace: kserve-ci-e2e-test resourceVersion: "14058" uid: fef1ce21-b60d-4007-83a3-ca9b24b30e0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9689de6ed3db8 namespace: kserve-ci-e2e-test resourceVersion: "14063" uid: 7a58a15a-1687-4d1a-90ce-4cd9d349c189 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r namespace: kserve-ci-e2e-test resourceVersion: "13056" uid: 5f7de5da-b616-41d1-90a8-4c2f4e9f81e7 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:57Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r.18a9689e7e2cdf98 namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: 5ee8162c-b17c-4f84-92c0-fd619558cb36 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9db4b-predictor-7ffc8c88fb namespace: kserve-ci-e2e-test resourceVersion: "13044" uid: 8999e522-0555-4850-ac0b-926a1c083f06 kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: 'Created pod: error-404-isvc-9db4b-predictor-7ffc8c88fb-lb79r' metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:25Z" name: error-404-isvc-9db4b-predictor-7ffc8c88fb.18a96888fa391363 namespace: kserve-ci-e2e-test resourceVersion: "13055" uid: 5e70d4e5-b42b-46c1-a005-554d4abe46a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13042" uid: 8cf2e61b-47ab-4207-b140-a09ab75ff292 kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Scaled up replica set error-404-isvc-9db4b-predictor-7ffc8c88fb from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:25Z" name: error-404-isvc-9db4b-predictor.18a96888f3929642 namespace: kserve-ci-e2e-test resourceVersion: "13047" uid: 40d7b98f-0da3-4ac8-ae03-7ebd84473af9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13043" uid: 50b4fe11-24e5-40bc-a3e3-ce6b35bb3d33 kind: Event lastTimestamp: "2026-04-24T21:35:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:10Z" name: error-404-isvc-9db4b-predictor.18a9688c72579fe1 namespace: kserve-ci-e2e-test resourceVersion: "13572" uid: 5b916da8-c0ce-4962-a82e-effd3aa0581d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13043" uid: 50b4fe11-24e5-40bc-a3e3-ce6b35bb3d33 kind: Event lastTimestamp: "2026-04-24T21:35:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:10Z" name: error-404-isvc-9db4b-predictor.18a9688c72ca0224 namespace: kserve-ci-e2e-test resourceVersion: "13573" uid: cecbcbb4-d62f-4aa3-b49c-08075702d592 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13265" uid: 50b4fe11-24e5-40bc-a3e3-ce6b35bb3d33 kind: Event lastTimestamp: "2026-04-24T21:35:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:40Z" name: error-404-isvc-9db4b-predictor.18a96896f111426f namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: 755673ae-00e5-4cc6-8c80-f9696de51733 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13265" uid: 50b4fe11-24e5-40bc-a3e3-ce6b35bb3d33 kind: Event lastTimestamp: "2026-04-24T21:35:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:40Z" name: error-404-isvc-9db4b-predictor.18a96896f1ab7312 namespace: kserve-ci-e2e-test resourceVersion: "13831" uid: faf02788-a41c-40be-a76f-b59599297e0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9db4b namespace: kserve-ci-e2e-test resourceVersion: "13019" uid: 68293d88-06d9-47c3-8b37-eb8c312bada0 kind: Event lastTimestamp: "2026-04-24T21:35:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:33Z" name: error-404-isvc-9db4b.18a96888f3518c67 namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: 81b99476-3b87-4944-9f8d-403c5bafa0c2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9db4b namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: 68293d88-06d9-47c3-8b37-eb8c312bada0 kind: Event lastTimestamp: "2026-04-24T21:35:32Z" message: InferenceService [error-404-isvc-9db4b] is Ready metadata: creationTimestamp: "2026-04-24T21:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:32Z" name: error-404-isvc-9db4b.18a96898b1680252 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 273580e3-34a0-43cd-855a-30e45b5710f2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:46:20.687350Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:46:20Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692f7ee0021e namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 8e0702a4-1eca-4b0b-8893-1f839eb69a86 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a9fd5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:20Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692f8cf82984 namespace: kserve-ci-e2e-test resourceVersion: "19419" uid: 504c7f86-af99-4131-8b98-472e013beb47 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:46:21Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692fbb6db2f9 namespace: kserve-ci-e2e-test resourceVersion: "19438" uid: b172a78f-5749-43d7-85c4-fee2539548f7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692fbcc9d6ca namespace: kserve-ci-e2e-test resourceVersion: "19440" uid: de96a50f-4f16-4815-a716-6802b879b5ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692fc03160bb namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: 53a5765d-c000-481c-9e46-28c43047b084 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692fc0a11da3 namespace: kserve-ci-e2e-test resourceVersion: "19443" uid: f763bac5-8418-4896-820a-d4efc81383e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692fc0acad24 namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: af4bf92a-a7b9-47dc-bb9d-362ce6275d77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692fc3939eff namespace: kserve-ci-e2e-test resourceVersion: "19445" uid: 0a3947f1-ef73-431e-a0f3-d04810947128 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692fc406c07d namespace: kserve-ci-e2e-test resourceVersion: "19446" uid: 85353c79-4731-4246-bd40-94adda96d552 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:46:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:46:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:59Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a9692fcdfe50dd namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 09c0db20-4557-45aa-91e2-1bf83998ea66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:35Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a969b0b4b077ca namespace: kserve-ci-e2e-test resourceVersion: "23768" uid: 5cd3104c-612f-46b3-a1e3-1e6a16a3d844 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 989e357d-91ff-427f-a266-dcbde106f0fa kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:35Z" name: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5.18a969b0b4b09213 namespace: kserve-ci-e2e-test resourceVersion: "23773" uid: 9fde4f77-0385-4348-99f2-064067b44f31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a9fd5-predictor-869bc478f6 namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: 244bdc0b-e410-468a-9cc4-8897f500ca3d kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: 'Created pod: error-404-isvc-a9fd5-predictor-869bc478f6-jwtd5' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:20Z" name: error-404-isvc-a9fd5-predictor-869bc478f6.18a9692f7e49a020 namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: 4fce67f8-7cb8-4a5f-ad65-8336df2f6de5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19393" uid: 3ba7ad46-7e18-47e3-9f66-86898cb6803c kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: Scaled up replica set error-404-isvc-a9fd5-predictor-869bc478f6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:20Z" name: error-404-isvc-a9fd5-predictor.18a9692f7c98c1be namespace: kserve-ci-e2e-test resourceVersion: "19397" uid: 7af73966-ca77-421d-9863-27edb0c51d5c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: 2cab1537-e995-4a59-92c1-377b55e1ab42 kind: Event lastTimestamp: "2026-04-24T21:46:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:35Z" name: error-404-isvc-a9fd5-predictor.18a96932fb63d907 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: aaadf0bc-a2c5-4684-a147-7feed5f8c6cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: 2cab1537-e995-4a59-92c1-377b55e1ab42 kind: Event lastTimestamp: "2026-04-24T21:46:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:35Z" name: error-404-isvc-a9fd5-predictor.18a96932fbd7c302 namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: fb8e9f65-9130-4789-bd00-c04a2571dc61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19588" uid: 2cab1537-e995-4a59-92c1-377b55e1ab42 kind: Event lastTimestamp: "2026-04-24T21:47:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:35Z" name: error-404-isvc-a9fd5-predictor.18a969367acaff60 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: d3052f2b-5ffc-4ff4-92da-073c3b07feac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19588" uid: 2cab1537-e995-4a59-92c1-377b55e1ab42 kind: Event lastTimestamp: "2026-04-24T21:47:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:35Z" name: error-404-isvc-a9fd5-predictor.18a969367b3ed8df namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 70ba9bcb-7f8a-4b01-973f-195a75939037 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: c08fb5d3-43f8-4a7d-bd0b-d8cf110f0851 kind: Event lastTimestamp: "2026-04-24T21:47:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:09Z" name: error-404-isvc-a9fd5.18a9692f7c9d6d1c namespace: kserve-ci-e2e-test resourceVersion: "20077" uid: a2e78340-2092-40c8-8db9-d679ed6f9595 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: c08fb5d3-43f8-4a7d-bd0b-d8cf110f0851 kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-a9fd5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9fd5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:20Z" name: error-404-isvc-a9fd5.18a9692f7d19dc61 namespace: kserve-ci-e2e-test resourceVersion: "19400" uid: f0bba7ac-4364-4628-88a8-78e2048c027e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: c08fb5d3-43f8-4a7d-bd0b-d8cf110f0851 kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9fd5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:20Z" name: error-404-isvc-a9fd5.18a9692f7d1a2ddb namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: 41947dc7-d2a8-4a30-9b42-090c41cc5745 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: c08fb5d3-43f8-4a7d-bd0b-d8cf110f0851 kind: Event lastTimestamp: "2026-04-24T21:47:09Z" message: InferenceService [error-404-isvc-a9fd5] is Ready metadata: creationTimestamp: "2026-04-24T21:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:09Z" name: error-404-isvc-a9fd5.18a9693ac90ca967 namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: 5b1043c6-e1a2-442a-a593-22530583dc32 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:47:00.533038Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d1134-predictor-796fc776b4-5ssqh to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:47:00Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a96938c5dd9a86 namespace: kserve-ci-e2e-test resourceVersion: "19935" uid: deabdcf0-c264-46d2-b0c7-0ab2c9684dbc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d1134-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a96938d37bf711 namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: d1526160-6ba1-4f7e-83aa-bfbec8137dca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19945" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:47:01Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:47:01Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a9693902d8caa5 namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: 60a85a42-f49a-4831-ba91-f7d31ef2c954 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:47:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:01Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a969390552b926 namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: 91539386-0b53-4e81-85ca-c59b09b65469 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:47:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:01Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a9693908deffa3 namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: f09fb803-b492-452a-9000-ec20b0dca587 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:47:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:01Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a9693909509bc0 namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: 04a18c69-7f63-4056-8833-694e612849d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:47:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:01Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a96939095bc8fa namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: 26244b40-0f99-436f-9711-a0beaa42a418 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:47:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:01Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a969390c3caa5f namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: 928bc292-4539-40d6-8c48-bc4d3366fc4d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:47:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:01Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a969390ce6575d namespace: kserve-ci-e2e-test resourceVersion: "19984" uid: 8bc0abd9-21d3-4906-ab0c-da1136b47ec4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:56:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:18Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a9693926dc4cec namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 70744842-9ce4-42ef-babc-676d0f479190 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a969b9eb877911 namespace: kserve-ci-e2e-test resourceVersion: "24171" uid: 7e86bd47-ab30-44b4-88e5-bec5c3704f62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a969b9eb885f7d namespace: kserve-ci-e2e-test resourceVersion: "24174" uid: f0eddb08-9102-482f-9fad-92ec5327b35f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: 303b11b3-a44c-4cd3-896a-000a60c34a09 kind: Event lastTimestamp: "2026-04-24T21:56:18Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:18Z" name: error-404-isvc-d1134-predictor-796fc776b4-5ssqh.18a969ba9b8a54ec namespace: kserve-ci-e2e-test resourceVersion: "24287" uid: 85bb2beb-a3a0-4b64-9276-a11ba0036bc2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d1134-predictor-796fc776b4 namespace: kserve-ci-e2e-test resourceVersion: "19913" uid: 8c54d914-194d-47df-b7eb-b475283e86cc kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: 'Created pod: error-404-isvc-d1134-predictor-796fc776b4-5ssqh' metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:00Z" name: error-404-isvc-d1134-predictor-796fc776b4.18a96938c57a2970 namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: 9d507060-b0dc-4616-aa1f-d2713099759e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 7f364dce-3776-49c8-a630-ab93e6242b10 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Scaled up replica set error-404-isvc-d1134-predictor-796fc776b4 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:00Z" name: error-404-isvc-d1134-predictor.18a96938c38d4572 namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: 13520343-b46b-4c66-bad8-ce39f7d91a3f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:47:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "19912" uid: 9791daf4-6116-49e6-8997-1a59741c07cb kind: Event lastTimestamp: "2026-04-24T21:47:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:45Z" name: error-404-isvc-d1134-predictor.18a9693c42021694 namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: d89fc20b-8a1b-4570-9b32-c41d2e4777ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:47:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "19912" uid: 9791daf4-6116-49e6-8997-1a59741c07cb kind: Event lastTimestamp: "2026-04-24T21:47:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:45Z" name: error-404-isvc-d1134-predictor.18a9693c4254364d namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 2bfe9067-3ca5-4519-bc2e-148454d26f97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:48:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 9791daf4-6116-49e6-8997-1a59741c07cb kind: Event lastTimestamp: "2026-04-24T21:48:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:15Z" name: error-404-isvc-d1134-predictor.18a96946bebafa03 namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 95b894ad-9c12-4e68-8542-8ba718b406f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:48:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 9791daf4-6116-49e6-8997-1a59741c07cb kind: Event lastTimestamp: "2026-04-24T21:48:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:15Z" name: error-404-isvc-d1134-predictor.18a96946bf2b6cea namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: 075d5155-89d0-4e5a-81ac-a6fe16192901 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1134 namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: fc96fcf0-eef6-4a2d-8b4c-bdadb73e2133 kind: Event lastTimestamp: "2026-04-24T21:47:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:48Z" name: error-404-isvc-d1134.18a96938c35e0d68 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 45f0d62d-aa9e-41df-9279-88c445d97677 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1134 namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: fc96fcf0-eef6-4a2d-8b4c-bdadb73e2133 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-d1134": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d1134": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:00Z" name: error-404-isvc-d1134.18a96938c3c4f855 namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: 87675872-1f92-410d-880c-08ae8c3e54ef reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1134 namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: fc96fcf0-eef6-4a2d-8b4c-bdadb73e2133 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d1134": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:00Z" name: error-404-isvc-d1134.18a96938c3c55181 namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: 5fe2a8a8-e0ea-46b1-9aad-606d2f7d130d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1134 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: fc96fcf0-eef6-4a2d-8b4c-bdadb73e2133 kind: Event lastTimestamp: "2026-04-24T21:47:48Z" message: InferenceService [error-404-isvc-d1134] is Ready metadata: creationTimestamp: "2026-04-24T21:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:48Z" name: error-404-isvc-d1134.18a96943e4951f14 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 2de9add5-95b2-4e6b-8844-787853c4c5a7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:45:10.463001Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18504" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9691f252d7197 namespace: kserve-ci-e2e-test resourceVersion: "18508" uid: e328a59b-0b5b-40f1-bcba-d41b6560f93f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18513" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9691f3e474d9f namespace: kserve-ci-e2e-test resourceVersion: "18529" uid: 9e764b02-6f20-42d7-9e0e-46e666bf4e22 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9691f3f979e71 namespace: kserve-ci-e2e-test resourceVersion: "18531" uid: 21ceabe1-01bb-4b31-9963-ff30a3309eb4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9691f42c29cc2 namespace: kserve-ci-e2e-test resourceVersion: "18534" uid: 4d5d8b34-16d3-4b92-bdb4-153322615a2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9691f4333affb namespace: kserve-ci-e2e-test resourceVersion: "18535" uid: 58589e09-5dd8-4714-a0c7-31c25f6afd17 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9691f433e96e2 namespace: kserve-ci-e2e-test resourceVersion: "18536" uid: dcd85724-f31f-4902-90fc-47b535aace1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:45:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:11Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9691f460dc7f8 namespace: kserve-ci-e2e-test resourceVersion: "18537" uid: f85276d4-e888-4313-a68c-3e681674690e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:45:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:11Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9691f469ae5d9 namespace: kserve-ci-e2e-test resourceVersion: "18538" uid: 4e568df8-abab-4d6e-88a4-6919c6b294e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:45:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:48Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9691fae6a0cd0 namespace: kserve-ci-e2e-test resourceVersion: "18853" uid: 34a284f6-4322-426b-a4ff-955d9ecffe6c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:20Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9692f7b9a5189 namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: e1dd179b-2be5-4a6b-876d-34082824e01a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:20Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9692f7b9ad149 namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: 5f5790e1-5e7a-499e-952e-b9573b741b15 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8cb4eb76-33f9-42a7-94bd-501aeeab32f1 kind: Event lastTimestamp: "2026-04-24T21:46:23Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:23Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt.18a9693036aa4be8 namespace: kserve-ci-e2e-test resourceVersion: "19471" uid: e6bd7590-edf4-4b68-a0b6-eb23cbb651c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dd3fc-predictor-5bbf46867d namespace: kserve-ci-e2e-test resourceVersion: "18494" uid: 6e1de252-07ed-49f0-b3b0-4cde1f028a59 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: 'Created pod: error-404-isvc-dd3fc-predictor-5bbf46867d-gzdnt' metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc-predictor-5bbf46867d.18a9691f24ca1190 namespace: kserve-ci-e2e-test resourceVersion: "18506" uid: 4d3fda17-cba5-426f-a0da-e83f3abbd169 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18490" uid: 3ac026d1-ef79-4e1c-9f50-86e139057f72 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Scaled up replica set error-404-isvc-dd3fc-predictor-5bbf46867d from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc-predictor.18a9691f222ddb86 namespace: kserve-ci-e2e-test resourceVersion: "18498" uid: aebb6cab-6713-4d20-a081-9b931a2361c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18493" uid: e2aaa103-32a7-47d6-9788-3f2bdad7d2df kind: Event lastTimestamp: "2026-04-24T21:45:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:40Z" name: error-404-isvc-dd3fc-predictor.18a96922a0a73c18 namespace: kserve-ci-e2e-test resourceVersion: "18787" uid: 3eeaa0cd-766b-4330-809c-8a6ddc906e21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18493" uid: e2aaa103-32a7-47d6-9788-3f2bdad7d2df kind: Event lastTimestamp: "2026-04-24T21:45:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:40Z" name: error-404-isvc-dd3fc-predictor.18a96922a10c3e85 namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: 9c7fe919-8a29-4e30-8bfa-a76183a6ac7b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: e2aaa103-32a7-47d6-9788-3f2bdad7d2df kind: Event lastTimestamp: "2026-04-24T21:46:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:10Z" name: error-404-isvc-dd3fc-predictor.18a969299e8c041d namespace: kserve-ci-e2e-test resourceVersion: "19218" uid: 205254f5-2dc9-4173-9647-9d1603c483d0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: e2aaa103-32a7-47d6-9788-3f2bdad7d2df kind: Event lastTimestamp: "2026-04-24T21:46:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:10Z" name: error-404-isvc-dd3fc-predictor.18a969299efb1a76 namespace: kserve-ci-e2e-test resourceVersion: "19219" uid: 2922e0fc-db19-44df-b7ee-9d0cb61b3964 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "18462" uid: 0eedfb5a-66b2-4b20-804b-3bc8f8c41f5c kind: Event lastTimestamp: "2026-04-24T21:45:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:58Z" name: error-404-isvc-dd3fc.18a9691f22015f6a namespace: kserve-ci-e2e-test resourceVersion: "19093" uid: 9ccfe5df-a77f-4b2e-97f7-81dfe9f031c4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "18462" uid: 0eedfb5a-66b2-4b20-804b-3bc8f8c41f5c kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-dd3fc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dd3fc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc.18a9691f22732e63 namespace: kserve-ci-e2e-test resourceVersion: "18501" uid: 560b2020-847b-40ef-b1aa-16625481d523 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "18462" uid: 0eedfb5a-66b2-4b20-804b-3bc8f8c41f5c kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dd3fc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:10Z" name: error-404-isvc-dd3fc.18a9691f22738e8b namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: d01d7819-ce4f-4190-9531-ec7836139edf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 0eedfb5a-66b2-4b20-804b-3bc8f8c41f5c kind: Event lastTimestamp: "2026-04-24T21:45:58Z" message: InferenceService [error-404-isvc-dd3fc] is Ready metadata: creationTimestamp: "2026-04-24T21:45:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:58Z" name: error-404-isvc-dd3fc.18a9692a6c1e2c4a namespace: kserve-ci-e2e-test resourceVersion: "19091" uid: ffa2ebd2-f903-4d57-95e0-d0512dd91d8e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:56:15.406002Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24226" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fc125-predictor-7967db9f76-wrnsk to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a969b9f6de79f7 namespace: kserve-ci-e2e-test resourceVersion: "24230" uid: ec1621b5-8610-45ce-82a7-757f60747372 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24236" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a969ba103c19ea namespace: kserve-ci-e2e-test resourceVersion: "24251" uid: dd6ed74c-d5b4-4ceb-8a95-b9fc93edf65f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a969ba11aa123d namespace: kserve-ci-e2e-test resourceVersion: "24253" uid: 66b9a2a4-8313-4f19-9278-5cd3427f68fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a969ba15b6d0fb namespace: kserve-ci-e2e-test resourceVersion: "24257" uid: 75d5c9f8-1f82-49c7-ab35-bee124460663 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a969ba1647de00 namespace: kserve-ci-e2e-test resourceVersion: "24258" uid: 08b86d25-d016-4e95-8c23-bc33b8b9b4c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a969ba16543859 namespace: kserve-ci-e2e-test resourceVersion: "24259" uid: cfe43194-8743-47b2-90ec-b377827eb015 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a969ba192c3cd8 namespace: kserve-ci-e2e-test resourceVersion: "24260" uid: b5fc7021-1fe9-4677-9496-ec4fa56e697d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a969ba199ba1da namespace: kserve-ci-e2e-test resourceVersion: "24261" uid: 05b9a654-4933-4b50-b8cf-216baf649a0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:56:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T21:56:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:53Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a969ba87231882 namespace: kserve-ci-e2e-test resourceVersion: "24786" uid: f174c4d5-4c0e-4d00-8e99-09bbb7efc3e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T22:13:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:13:34Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a96aabfae08561 namespace: kserve-ci-e2e-test resourceVersion: "32206" uid: 15ed77fe-a064-4540-b614-6a678aece0ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: f13ffeb5-f1e9-487a-946d-0c5a5d04cb3d kind: Event lastTimestamp: "2026-04-24T22:13:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:13:34Z" name: error-404-isvc-fc125-predictor-7967db9f76-wrnsk.18a96aabfae0fd0f namespace: kserve-ci-e2e-test resourceVersion: "32207" uid: 9f770922-cd4b-4299-95da-0974a52e786c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fc125-predictor-7967db9f76 namespace: kserve-ci-e2e-test resourceVersion: "24218" uid: 55a66db6-d345-42d1-8988-370471d4c19c kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'Created pod: error-404-isvc-fc125-predictor-7967db9f76-wrnsk' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor-7967db9f76.18a969b9f677aac9 namespace: kserve-ci-e2e-test resourceVersion: "24227" uid: 103a3f82-1151-4780-9145-2253a4fae749 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24216" uid: ac999933-6e7f-488d-9fe0-287dfbd953c0 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Scaled up replica set error-404-isvc-fc125-predictor-7967db9f76 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125-predictor.18a969b9f45874d9 namespace: kserve-ci-e2e-test resourceVersion: "24220" uid: af8adca2-1c38-47d5-997e-3daaaca9234a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:56:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24217" uid: 007a2a27-7ba4-4651-829e-576e0738efb5 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:56:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:45Z" name: error-404-isvc-fc125-predictor.18a969bd72e69e6a namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 5cd7744f-13c9-4e00-ae02-1b376257de1f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:56:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24217" uid: 007a2a27-7ba4-4651-829e-576e0738efb5 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:56:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:45Z" name: error-404-isvc-fc125-predictor.18a969bd7339c080 namespace: kserve-ci-e2e-test resourceVersion: "24576" uid: 92498e65-1923-4d07-817c-c102b118f087 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:57:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24416" uid: 007a2a27-7ba4-4651-829e-576e0738efb5 kind: Event lastTimestamp: "2026-04-24T21:57:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:30Z" name: error-404-isvc-fc125-predictor.18a969c470cfd391 namespace: kserve-ci-e2e-test resourceVersion: "25138" uid: a3d4724d-7aa1-4de9-abc2-5ec318ef76e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:57:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24416" uid: 007a2a27-7ba4-4651-829e-576e0738efb5 kind: Event lastTimestamp: "2026-04-24T21:57:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:30Z" name: error-404-isvc-fc125-predictor.18a969c4715b0f39 namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: b798becb-f02d-4db0-bed4-5bd76c6db652 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24185" uid: 932cdf67-6314-4e78-bee0-4a911ac13cf5 kind: Event lastTimestamp: "2026-04-24T21:57:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:03Z" name: error-404-isvc-fc125.18a969b9f4370199 namespace: kserve-ci-e2e-test resourceVersion: "24895" uid: 34067c78-3384-4a0d-996b-5f5c2183d9c5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24185" uid: 932cdf67-6314-4e78-bee0-4a911ac13cf5 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-fc125": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fc125": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125.18a969b9f4aefc33 namespace: kserve-ci-e2e-test resourceVersion: "24222" uid: e613785d-5802-4f8e-b3f3-3a33d7b074fb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24185" uid: 932cdf67-6314-4e78-bee0-4a911ac13cf5 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fc125": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:15Z" name: error-404-isvc-fc125.18a969b9f4af5c90 namespace: kserve-ci-e2e-test resourceVersion: "24223" uid: 35555afd-d944-474a-8ac2-4b3851ccd8ea reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24893" uid: 932cdf67-6314-4e78-bee0-4a911ac13cf5 kind: Event lastTimestamp: "2026-04-24T21:57:03Z" message: InferenceService [error-404-isvc-fc125] is Ready metadata: creationTimestamp: "2026-04-24T21:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:03Z" name: error-404-isvc-fc125.18a969c54271b061 namespace: kserve-ci-e2e-test resourceVersion: "24894" uid: c1c14383-7f79-41a4-8343-86b1b596617f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:34:25.210868Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13000" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:34:25Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a96888e92745aa namespace: kserve-ci-e2e-test resourceVersion: "13007" uid: 4c4a9dc2-e92d-4778-b51f-1c94365e4c22 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13016" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:34:25Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688901ef5523 namespace: kserve-ci-e2e-test resourceVersion: "13063" uid: 298a3a82-c3a5-4c33-a08d-ab7d063b5285 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:25Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a96889043fb266 namespace: kserve-ci-e2e-test resourceVersion: "13065" uid: bc1f2281-4dc7-4c07-b042-50aaf90ff28b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" in 16.633s (16.633s including waiting). Image size: 299845049 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688ce3a9534f namespace: kserve-ci-e2e-test resourceVersion: "13280" uid: bf14e2d5-8348-4d43-9083-d2a93005e167 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688ce96a57c7 namespace: kserve-ci-e2e-test resourceVersion: "13283" uid: 85b556b1-7c54-4cca-beb2-c28c1cac462b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688ce9dc1cec namespace: kserve-ci-e2e-test resourceVersion: "13284" uid: 3fed76dd-4e43-4470-991c-60d442d103be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:46Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" metadata: creationTimestamp: "2026-04-24T21:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:46Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688df61b6db2 namespace: kserve-ci-e2e-test resourceVersion: "13347" uid: 4049f4d1-287a-41fb-8138-45aebfd575d0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" in 6.142s (6.142s including waiting). Image size: 1560926126 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688f64408a44 namespace: kserve-ci-e2e-test resourceVersion: "13413" uid: 449d1dcf-b162-4d75-9e0f-de7c50472294 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688f6b263d03 namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: d34abf95-f528-4a0a-9b4c-5bed399ebe5a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688f6c949d1d namespace: kserve-ci-e2e-test resourceVersion: "13417" uid: e4875e55-adf0-433a-b110-277cca198f7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688f6ca99265 namespace: kserve-ci-e2e-test resourceVersion: "13418" uid: 0eec6d80-9f7d-44ac-8327-ecd98eb69545 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688f726b08c4 namespace: kserve-ci-e2e-test resourceVersion: "13420" uid: 753354bb-a05b-4265-b6b6-26e862e7ea99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688f73b5ed01 namespace: kserve-ci-e2e-test resourceVersion: "13421" uid: 74e22d1c-bc6d-42ab-afd1-9a7c4a9727dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:34:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:35:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:50Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a9688fd4af0dd3 namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: 2e51a0d5-f9fd-4aa1-b834-b6dadf6b07ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:35Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a968a73b9da8f5 namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: a2f9df13-45eb-478b-817b-0984e2bcee28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:35Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a968a73b9e1e31 namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: e63080e2-31a9-4a31-a6ef-93d75993d7d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f namespace: kserve-ci-e2e-test resourceVersion: "13004" uid: d9dd38aa-c701-426e-aa68-982add5b0621 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:35Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f.18a968a758e99e5e namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 64d43731-d444-4b14-998a-b7271e034fd6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-58d89b86bf namespace: kserve-ci-e2e-test resourceVersion: "12989" uid: 880ea264-fdb7-43d7-bfb4-7eac7f39ebaa kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-58d89b86bf-hch9f' metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:25Z" name: isvc-sklearn-graph-1-predictor-58d89b86bf.18a96888e8a25c24 namespace: kserve-ci-e2e-test resourceVersion: "13003" uid: 0f49e635-2787-468f-a2e8-deaa8834bc51 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12987" uid: f507e5e3-b9c3-4f49-9fc8-77bd3abbd7cc kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-58d89b86bf from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:25Z" name: isvc-sklearn-graph-1-predictor.18a96888e3df8b82 namespace: kserve-ci-e2e-test resourceVersion: "12992" uid: ec99dbf0-e1fb-4684-93d7-4b29dcd1c4a4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12988" uid: 22789d9d-332d-434c-9fbd-677257c04fe8 kind: Event lastTimestamp: "2026-04-24T21:35:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:25Z" name: isvc-sklearn-graph-1-predictor.18a9688c626f3b6d namespace: kserve-ci-e2e-test resourceVersion: "13693" uid: 1f96b331-c08d-49cd-a22d-fe29c0141032 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12988" uid: 22789d9d-332d-434c-9fbd-677257c04fe8 kind: Event lastTimestamp: "2026-04-24T21:35:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:25Z" name: isvc-sklearn-graph-1-predictor.18a9688c62de28b8 namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: bb0aab6c-61f7-4685-af1e-9f8d1d874b85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:35:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13255" uid: 22789d9d-332d-434c-9fbd-677257c04fe8 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: isvc-sklearn-graph-1-predictor.18a9689360e34af0 namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: cdfde914-2cbb-495c-ba6c-752e9c36aaea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:35:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13255" uid: 22789d9d-332d-434c-9fbd-677257c04fe8 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: isvc-sklearn-graph-1-predictor.18a96893618c60b1 namespace: kserve-ci-e2e-test resourceVersion: "14428" uid: 069b36e2-545b-4475-add6-fe2e6eec4a65 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "12971" uid: 820aa8dd-dcde-48d5-9d6f-0721f64e499c kind: Event lastTimestamp: "2026-04-24T21:36:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:01Z" name: isvc-sklearn-graph-1.18a96888e3b4236b namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: 4d4239a1-7ca1-4e39-b4c3-f2258c967d18 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: 820aa8dd-dcde-48d5-9d6f-0721f64e499c kind: Event lastTimestamp: "2026-04-24T21:36:01Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-24T21:36:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:01Z" name: isvc-sklearn-graph-1.18a9689f397f2657 namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: 7c36b527-38f3-4af1-a77b-4c2a50024b69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:34:26.194812Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13096" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw to ip-10-0-130-31.ec2.internal metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:34:26Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688923cd12a0 namespace: kserve-ci-e2e-test resourceVersion: "13100" uid: 90ae81ac-8dd1-4972-ad5c-d4a2a50b932e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13104" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:26Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:34:26Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a968893e004e7d namespace: kserve-ci-e2e-test resourceVersion: "13112" uid: 8cd5700d-41a1-452f-8e79-4056d2e75a00 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:26Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:26Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a96889402e5e5c namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: fcd22edc-2dbf-44ec-b2ef-84ea767a9e2a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:30Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" in 3.387s (3.387s including waiting). Image size: 299845049 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:30Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688a0a116c64 namespace: kserve-ci-e2e-test resourceVersion: "13153" uid: f98c9ad1-97c0-4afc-ae1d-78b50f1f5120 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:30Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688a0d728d22 namespace: kserve-ci-e2e-test resourceVersion: "13154" uid: f0687b80-05bc-412a-8cda-42b5f7312ad9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:30Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688a0def3d77 namespace: kserve-ci-e2e-test resourceVersion: "13155" uid: 4ca8657c-3d9f-4ccb-be2a-8061e4a1088e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:33Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" metadata: creationTimestamp: "2026-04-24T21:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:33Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688af4bd996f namespace: kserve-ci-e2e-test resourceVersion: "13199" uid: bf58675a-34a5-4a92-8aa6-19a6a9a65952 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:47Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1450" in 13.373s (13.373s including waiting). Image size: 1560926126 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:47Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688e11da2bcd namespace: kserve-ci-e2e-test resourceVersion: "13355" uid: c7f6a641-5896-4684-9476-fd0c6f41fb24 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:47Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688e15393102 namespace: kserve-ci-e2e-test resourceVersion: "13356" uid: e2bdbb42-f42a-4e72-8916-d5cf9d774aae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:47Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688e15b1b94e namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 95b75590-25eb-4146-ae89-4572a304cc24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:47Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:47Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688e15bfe48e namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: 9e8f102f-e2aa-45da-90dc-64959c5e387d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:49Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.222s (2.222s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:49Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688e9a37a2d6 namespace: kserve-ci-e2e-test resourceVersion: "13376" uid: dd232e74-4234-4930-a047-866a7e7298a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:34:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:49Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688e9d3bb8f8 namespace: kserve-ci-e2e-test resourceVersion: "13377" uid: bd4c1265-f76e-411e-9809-ab1a285414d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:34:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:34:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:49Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688e9dc540e8 namespace: kserve-ci-e2e-test resourceVersion: "13378" uid: a566b7d3-5473-4a39-ad8e-44a74283feab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:34:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:36:37Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:34:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:37Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a9688eee21abb2 namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: 09b17f18-a061-44e9-b304-23fa4f1e89be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:35Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a968a732a49100 namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 0a56cd57-ccf9-4657-b7f6-6de7b3838d36 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:35Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a968a732a55d81 namespace: kserve-ci-e2e-test resourceVersion: "14593" uid: dd511020-cc91-4d21-91bc-db0fc6a91938 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw namespace: kserve-ci-e2e-test resourceVersion: "13098" uid: 721f5134-e6d0-43ca-865b-ae193260f5a3 kind: Event lastTimestamp: "2026-04-24T21:36:37Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:37Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw.18a968a79c5e6416 namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: a4c1cfb0-1f40-442f-8eed-4224f9e6cc0e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-31.ec2.internal source: component: kubelet host: ip-10-0-130-31.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-58cc99f6ff namespace: kserve-ci-e2e-test resourceVersion: "13081" uid: 2c58d95d-5bc9-4661-b54f-7590fa5fbc00 kind: Event lastTimestamp: "2026-04-24T21:34:26Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-58cc99f6ff-jrwtw' metadata: creationTimestamp: "2026-04-24T21:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:26Z" name: isvc-sklearn-graph-2-predictor-58cc99f6ff.18a96889235abc02 namespace: kserve-ci-e2e-test resourceVersion: "13097" uid: a68ea165-50a6-410c-9bff-f9481bdd30a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13079" uid: 4888d41c-9373-4921-8c78-a34bb75230d2 kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-58cc99f6ff from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:25Z" name: isvc-sklearn-graph-2-predictor.18a968890d6131d2 namespace: kserve-ci-e2e-test resourceVersion: "13083" uid: 00ee8e94-6a48-457e-94e5-244b02a926d2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13080" uid: 204d9537-6b6f-44d3-a0e7-985b8f63a034 kind: Event lastTimestamp: "2026-04-24T21:35:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:10Z" name: isvc-sklearn-graph-2-predictor.18a9688c8bea6bf8 namespace: kserve-ci-e2e-test resourceVersion: "13576" uid: cb28b9aa-fe67-46f2-b828-2ae47f226fe7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13080" uid: 204d9537-6b6f-44d3-a0e7-985b8f63a034 kind: Event lastTimestamp: "2026-04-24T21:35:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:10Z" name: isvc-sklearn-graph-2-predictor.18a9688c8c7e2fff namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: ec90ed08-29df-49ce-a2eb-a97dd9decc61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13269" uid: 204d9537-6b6f-44d3-a0e7-985b8f63a034 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: isvc-sklearn-graph-2-predictor.18a9689709cbc32d namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: eaab1935-7037-4f89-9426-e8dd08fe635f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13269" uid: 204d9537-6b6f-44d3-a0e7-985b8f63a034 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: isvc-sklearn-graph-2-predictor.18a968970a716800 namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: 3176c047-6afd-4805-a9d7-829cc8b4ef77 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13052" uid: 3bb6397c-9070-4b05-a7e9-1a56da8d29c0 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-2.18a968890d39784d namespace: kserve-ci-e2e-test resourceVersion: "14150" uid: 38a00a41-c429-4c36-bcab-738c36e1b152 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: 3bb6397c-9070-4b05-a7e9-1a56da8d29c0 kind: Event lastTimestamp: "2026-04-24T21:35:57Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-24T21:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:57Z" name: isvc-sklearn-graph-2.18a9689e515a1910 namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: e12ae605-8f8d-467b-9a33-d221fc5c24df reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:34:25.314053Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13020" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-n7ckc to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:34:25Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a96888ef4dc78d namespace: kserve-ci-e2e-test resourceVersion: "13027" uid: f1563c2a-13de-4603-877d-270a744643ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13036" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:34:25Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9688908e9cacd namespace: kserve-ci-e2e-test resourceVersion: "13068" uid: 668c9bc0-3dac-464d-b634-5d602f216f1b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:25Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a968890aa0ca1f namespace: kserve-ci-e2e-test resourceVersion: "13072" uid: 8af9f671-ec25-457c-a9c0-4e77c3eee5ea reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:29Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0dd2798f9cdbffc0563bd148612201df0e589ff8c26a6b19d321fd120fc5c097" in 4.174s (4.174s including waiting). Image size: 299845049 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9688a0379c195 namespace: kserve-ci-e2e-test resourceVersion: "13148" uid: 412537c4-e468-4f1b-b8c2-e76ddbb87cce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9688a0665e5c1 namespace: kserve-ci-e2e-test resourceVersion: "13149" uid: d6e94e80-d443-4c68-8bcc-4d97d76fd789 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9688a06dd7f1d namespace: kserve-ci-e2e-test resourceVersion: "13150" uid: c5fab36a-bece-44d6-aea0-cef5ca787ff4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:33Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T21:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:33Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9688af09d35eb namespace: kserve-ci-e2e-test resourceVersion: "13194" uid: 36625d86-33eb-4372-adae-e5df5bfcd45a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.626s (19.626s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9688f8273fc2f namespace: kserve-ci-e2e-test resourceVersion: "13424" uid: b0465c64-86c4-4a2c-94d7-48b0e7e9b9dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9688f8871e6a1 namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: 8b30cdc6-7542-4be0-a3b3-e71742849f85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9688f89de6d95 namespace: kserve-ci-e2e-test resourceVersion: "13426" uid: 2f294003-1f7e-4391-9f68-cc30e334387e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:53Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:53Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9688f89ebcee5 namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: 101be81e-1ab2-4d51-a0a9-213519fd38b7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:56Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.818s (2.818s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a9689031f18996 namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: a7cab048-1c6f-4f71-981e-99217cac553c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a96890379780b3 namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: d5221ea7-c398-44da-9d48-4162158285e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:34:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a96890387fd922 namespace: kserve-ci-e2e-test resourceVersion: "13462" uid: d84cc1cd-3a3e-4428-a7fe-94ea19274c24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:34:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:35:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:34:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:54Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a968908bc660d6 namespace: kserve-ci-e2e-test resourceVersion: "13968" uid: 5b1d9502-49e8-4e23-9d33-3577b388c3a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:35Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a968a72d9cada6 namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 1574a9a9-3059-4a3f-926e-f1ebadc9deeb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:35Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a968a72d9dfaf3 namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: 066ce17e-8134-4f10-822d-6e9fadee740a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc namespace: kserve-ci-e2e-test resourceVersion: "13022" uid: 4249ca4d-5581-4557-aeb2-69c27ee9c041 kind: Event lastTimestamp: "2026-04-24T21:36:40Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-04-24T21:36:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-n7ckc.18a968a84ba585c7 namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: c58b4a23-836c-4592-a0cf-230eb7041249 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "13008" uid: 63ddedd0-481c-45b5-86ef-8369c40a80bc kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-n7ckc' metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:25Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a96888ee9b3e8d namespace: kserve-ci-e2e-test resourceVersion: "13023" uid: 134813d2-cbca-47e5-a147-596637e146f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13005" uid: ace1050a-5f29-4353-a1e2-1c8043f28373 kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:25Z" name: isvc-xgboost-graph-predictor.18a96888e9f30a6c namespace: kserve-ci-e2e-test resourceVersion: "13012" uid: 824299fe-5eb3-4526-bd79-169bf65cd2d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13009" uid: ab560828-1a1a-4090-9fe8-90da596d6658 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: isvc-xgboost-graph-predictor.18a9688c68f9267c namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 609f2360-cea6-483d-8602-ba6c7c9c766f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:34:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13009" uid: ab560828-1a1a-4090-9fe8-90da596d6658 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: isvc-xgboost-graph-predictor.18a9688c69aaf562 namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: 8d6ac272-6ffc-412e-8485-1d72b050b661 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13258" uid: ab560828-1a1a-4090-9fe8-90da596d6658 kind: Event lastTimestamp: "2026-04-24T21:36:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:10Z" name: isvc-xgboost-graph-predictor.18a96896e6397615 namespace: kserve-ci-e2e-test resourceVersion: "14289" uid: 4801334f-5ed8-4e40-bd7a-b639d9f3da46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13258" uid: ab560828-1a1a-4090-9fe8-90da596d6658 kind: Event lastTimestamp: "2026-04-24T21:36:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:10Z" name: isvc-xgboost-graph-predictor.18a96896e6a6b6ac namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: 646c09b8-a8e8-43b9-b925-ac08c6c2dbd5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "12995" uid: d95c9fda-49d2-4398-a710-700dc2c7182b kind: Event lastTimestamp: "2026-04-24T21:36:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:04Z" name: isvc-xgboost-graph.18a96888ea4e1a51 namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: c1acc2d6-f961-4cfe-966c-478a7c74c166 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: d95c9fda-49d2-4398-a710-700dc2c7182b kind: Event lastTimestamp: "2026-04-24T21:36:04Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-24T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:04Z" name: isvc-xgboost-graph.18a9689fefa32d98 namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: a9e1059e-9616-4655-8d40-ab2e795ed6c8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:36:25.059489Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-85c5fc8d94-dvnrr namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 4d3b7b13-4461-4dac-a0c5-1d9e857b2ae6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-85c5fc8d94-dvnrr to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:36:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:36:25Z" name: model-chainer-85c5fc8d94-dvnrr.18a968a4d0b016cd namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: cd605e73-91b5-4a1c-9c69-8c0cd215db30 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:25Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-85c5fc8d94-dvnrr namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: 4d3b7b13-4461-4dac-a0c5-1d9e857b2ae6 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:36:25Z" name: model-chainer-85c5fc8d94-dvnrr.18a968a4e9facb08 namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: bef2638f-b9cf-48d5-8405-d104421ec819 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85c5fc8d94-dvnrr namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 4d3b7b13-4461-4dac-a0c5-1d9e857b2ae6 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:25Z" name: model-chainer-85c5fc8d94-dvnrr.18a968a4eb81bfdd namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: b0f47f0e-d064-453a-a5bb-df52efab728b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85c5fc8d94-dvnrr namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 4d3b7b13-4461-4dac-a0c5-1d9e857b2ae6 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-24T21:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:25Z" name: model-chainer-85c5fc8d94-dvnrr.18a968a4ee5460ed namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: a85c99c9-850e-4a1e-a342-3b1b50075d67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85c5fc8d94-dvnrr namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 4d3b7b13-4461-4dac-a0c5-1d9e857b2ae6 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-24T21:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:25Z" name: model-chainer-85c5fc8d94-dvnrr.18a968a4eed5590d namespace: kserve-ci-e2e-test resourceVersion: "14442" uid: 37dbcbd5-c0bb-4245-a76c-3b10af6b43f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85c5fc8d94-dvnrr namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 4d3b7b13-4461-4dac-a0c5-1d9e857b2ae6 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:35Z" name: model-chainer-85c5fc8d94-dvnrr.18a968a726874455 namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 81871377-3ef6-40d5-89ff-0fd814cfc17b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85c5fc8d94-dvnrr namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 4d3b7b13-4461-4dac-a0c5-1d9e857b2ae6 kind: Event lastTimestamp: "2026-04-24T21:37:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:02Z" name: model-chainer-85c5fc8d94-dvnrr.18a968a7a8b01330 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: 99bd9202-8851-439d-ae2b-2a317117d9f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-85c5fc8d94 namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 829c1c79-d0e4-4c30-80e7-8be723d0e6d5 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: 'Created pod: model-chainer-85c5fc8d94-dvnrr' metadata: creationTimestamp: "2026-04-24T21:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: model-chainer-85c5fc8d94.18a968a4cfbe612f namespace: kserve-ci-e2e-test resourceVersion: "14411" uid: 3a3bb145-e722-417b-baa8-dc3520bf6fce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: 69a1195b-279f-4f50-a913-ad3b9423636a kind: Event lastTimestamp: "2026-04-24T21:36:24Z" message: Scaled up replica set model-chainer-85c5fc8d94 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:25Z" name: model-chainer.18a968a4cce92feb namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: e28ca330-e0a1-4dc9-a1be-335a256fb736 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14421" uid: 5aee15e7-c303-41bb-88d1-346fa0d17438 kind: Event lastTimestamp: "2026-04-24T21:36:25Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-24T21:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:25Z" name: model-chainer.18a968a4d6701e77 namespace: kserve-ci-e2e-test resourceVersion: "14423" uid: 30082af8-8111-47cd-8a4b-4fd3854c1aeb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:37:35.281849Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-54155-84d8774cb-sx4nk namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: df15de12-ba86-43e8-bd9d-8cea0f40f72a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-54155-84d8774cb-sx4nk to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:37:35Z" name: sequence-graph-54155-84d8774cb-sx4nk.18a968b52a444425 namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: fdff31f0-3ea6-4416-930e-a38cb1d11f43 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-54155-84d8774cb-sx4nk namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: df15de12-ba86-43e8-bd9d-8cea0f40f72a kind: Event lastTimestamp: "2026-04-24T21:37:35Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:37:35Z" name: sequence-graph-54155-84d8774cb-sx4nk.18a968b544206964 namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: dc148e2a-6909-461f-928d-ad060e6d7ff1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-54155} kind: Pod name: sequence-graph-54155-84d8774cb-sx4nk namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: df15de12-ba86-43e8-bd9d-8cea0f40f72a kind: Event lastTimestamp: "2026-04-24T21:37:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:35Z" name: sequence-graph-54155-84d8774cb-sx4nk.18a968b54598315e namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: a60ea950-2009-4579-bd31-6ef96db52bb3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-54155} kind: Pod name: sequence-graph-54155-84d8774cb-sx4nk namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: df15de12-ba86-43e8-bd9d-8cea0f40f72a kind: Event lastTimestamp: "2026-04-24T21:37:35Z" message: 'Created container: sequence-graph-54155' metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:35Z" name: sequence-graph-54155-84d8774cb-sx4nk.18a968b5490ba854 namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: ce30c291-cd2a-4684-af4e-247410fd69b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-54155} kind: Pod name: sequence-graph-54155-84d8774cb-sx4nk namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: df15de12-ba86-43e8-bd9d-8cea0f40f72a kind: Event lastTimestamp: "2026-04-24T21:37:35Z" message: Started container sequence-graph-54155 metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:35Z" name: sequence-graph-54155-84d8774cb-sx4nk.18a968b5498626e2 namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: 900e4158-768e-4a4f-8a22-644e32738483 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-54155} kind: Pod name: sequence-graph-54155-84d8774cb-sx4nk namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: df15de12-ba86-43e8-bd9d-8cea0f40f72a kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Stopping container sequence-graph-54155 metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: sequence-graph-54155-84d8774cb-sx4nk.18a969285d867f3b namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 33538f24-c0ce-4404-8f8b-749fd96d105b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-54155} kind: Pod name: sequence-graph-54155-84d8774cb-sx4nk namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: df15de12-ba86-43e8-bd9d-8cea0f40f72a kind: Event lastTimestamp: "2026-04-24T21:46:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:17Z" name: sequence-graph-54155-84d8774cb-sx4nk.18a96928ee4070e9 namespace: kserve-ci-e2e-test resourceVersion: "19279" uid: 75027ef2-9eae-4cc8-b1b7-6e68ce6c25c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-54155-84d8774cb namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: 15eba30c-ef9c-4876-92ff-d9dbf18f299f kind: Event lastTimestamp: "2026-04-24T21:37:35Z" message: 'Created pod: sequence-graph-54155-84d8774cb-sx4nk' metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:35Z" name: sequence-graph-54155-84d8774cb.18a968b529e3b79f namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 948601d2-9796-4b33-91f0-e6c6ae442b92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-54155 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: ce110f7f-17f4-4ee3-ac91-6f4901311407 kind: Event lastTimestamp: "2026-04-24T21:37:35Z" message: Scaled up replica set sequence-graph-54155-84d8774cb from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:35Z" name: sequence-graph-54155.18a968b5290e168d namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: d57b292d-660a-4c91-8c85-9be0028b23cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-54155 namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 5ba495a3-dc53-48a1-89da-72eeccc71ad0 kind: Event lastTimestamp: "2026-04-24T21:37:35Z" message: InferenceGraph [sequence-graph-54155] is Ready metadata: creationTimestamp: "2026-04-24T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:35Z" name: sequence-graph-54155.18a968b52f244fbe namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 17f39796-0244-4ab6-8b56-37e5ec6117bf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-54155 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: dfe5dc01-3a71-4f25-b946-0b22fbcd4edf kind: Event lastTimestamp: "2026-04-24T21:37:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:50Z" name: sequence-graph-54155.18a968b8a8461159 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 5e5a13f1-3446-43af-a47a-204462d6d5e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-54155 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: dfe5dc01-3a71-4f25-b946-0b22fbcd4edf kind: Event lastTimestamp: "2026-04-24T21:37:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:50Z" name: sequence-graph-54155.18a968b8a89826c1 namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: 55b23e90-01b4-41ae-b661-2e8fa26e9fd0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-54155 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: dfe5dc01-3a71-4f25-b946-0b22fbcd4edf kind: Event lastTimestamp: "2026-04-24T21:38:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:05Z" name: sequence-graph-54155.18a968bc2757245e namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: d9d19d9b-44b6-471d-aec8-6150e30c6ba3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-54155 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: dfe5dc01-3a71-4f25-b946-0b22fbcd4edf kind: Event lastTimestamp: "2026-04-24T21:38:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:05Z" name: sequence-graph-54155.18a968bc27c14946 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 902c6a8e-770d-40a6-a1bf-e8ecff135ca6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:46:50.207878Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6445b-696c86c896-8k2tz namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 61d18dda-aba2-4aa4-a61b-5a66c6078382 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-6445b-696c86c896-8k2tz to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:46:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:46:50Z" name: sequence-graph-6445b-696c86c896-8k2tz.18a969365e6f6b02 namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: b4b50058-8c3f-4199-8217-3b802f09eb99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6445b-696c86c896-8k2tz namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 61d18dda-aba2-4aa4-a61b-5a66c6078382 kind: Event lastTimestamp: "2026-04-24T21:46:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-6445b-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:50Z" name: sequence-graph-6445b-696c86c896-8k2tz.18a96936674a1fbd namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: b450475d-e4ec-41b1-9856-68468935b52c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:51Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6445b-696c86c896-8k2tz namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: 61d18dda-aba2-4aa4-a61b-5a66c6078382 kind: Event lastTimestamp: "2026-04-24T21:46:51Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:46:51Z" name: sequence-graph-6445b-696c86c896-8k2tz.18a969369b318fc5 namespace: kserve-ci-e2e-test resourceVersion: "19760" uid: 15daf695-f920-4580-8c22-f83e14bd6c28 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6445b} kind: Pod name: sequence-graph-6445b-696c86c896-8k2tz namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 61d18dda-aba2-4aa4-a61b-5a66c6078382 kind: Event lastTimestamp: "2026-04-24T21:46:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:51Z" name: sequence-graph-6445b-696c86c896-8k2tz.18a969369c733090 namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: a8b0a501-94d9-4757-99f5-8e3842e31188 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6445b} kind: Pod name: sequence-graph-6445b-696c86c896-8k2tz namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 61d18dda-aba2-4aa4-a61b-5a66c6078382 kind: Event lastTimestamp: "2026-04-24T21:46:51Z" message: 'Created container: sequence-graph-6445b' metadata: creationTimestamp: "2026-04-24T21:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:51Z" name: sequence-graph-6445b-696c86c896-8k2tz.18a969369f4ec76c namespace: kserve-ci-e2e-test resourceVersion: "19763" uid: e1895fb2-01ca-411c-8371-3e6f3ebb758e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6445b} kind: Pod name: sequence-graph-6445b-696c86c896-8k2tz namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 61d18dda-aba2-4aa4-a61b-5a66c6078382 kind: Event lastTimestamp: "2026-04-24T21:46:51Z" message: Started container sequence-graph-6445b metadata: creationTimestamp: "2026-04-24T21:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:51Z" name: sequence-graph-6445b-696c86c896-8k2tz.18a969369fc3b43d namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: a8a9c393-377f-4f8e-a3a8-f6f7c46cef64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6445b} kind: Pod name: sequence-graph-6445b-696c86c896-8k2tz namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 61d18dda-aba2-4aa4-a61b-5a66c6078382 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Stopping container sequence-graph-6445b metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: sequence-graph-6445b-696c86c896-8k2tz.18a96938b7a61bf4 namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: 5df103c9-a3e2-4e36-bb78-8a2821762ee0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:47:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6445b} kind: Pod name: sequence-graph-6445b-696c86c896-8k2tz namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 61d18dda-aba2-4aa4-a61b-5a66c6078382 kind: Event lastTimestamp: "2026-04-24T21:47:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:28Z" name: sequence-graph-6445b-696c86c896-8k2tz.18a96939607c3268 namespace: kserve-ci-e2e-test resourceVersion: "20263" uid: 06d16bdd-6b65-4d98-8ebb-099e534d679f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-6445b-696c86c896 namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: 04a7b169-baf0-48bb-8a96-9bea6055ee70 kind: Event lastTimestamp: "2026-04-24T21:46:50Z" message: 'Created pod: sequence-graph-6445b-696c86c896-8k2tz' metadata: creationTimestamp: "2026-04-24T21:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:50Z" name: sequence-graph-6445b-696c86c896.18a969365e0ebe17 namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: 3ee446cc-c122-4ebb-80b2-44918570f62e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-6445b namespace: kserve-ci-e2e-test resourceVersion: "19713" uid: 275abb77-4512-4eb1-a80a-b46f5b833db2 kind: Event lastTimestamp: "2026-04-24T21:46:50Z" message: Scaled up replica set sequence-graph-6445b-696c86c896 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:50Z" name: sequence-graph-6445b.18a969365d6ad99f namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: f1b3080c-f734-43c3-8a9b-7ce39d08d602 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6445b namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 06f4c226-4caa-453d-a1a0-b004214c7d0b kind: Event lastTimestamp: "2026-04-24T21:46:50Z" message: InferenceGraph [sequence-graph-6445b] is Ready metadata: creationTimestamp: "2026-04-24T21:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:50Z" name: sequence-graph-6445b.18a9693662988c57 namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: db9e59dd-a684-482d-ab53-a25c28f44d3b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:48:00.484112Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d1134-69b44c49f9-tm66r namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: c0483f79-b854-42da-a3b9-ac80a7066269 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-d1134-69b44c49f9-tm66r to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:48:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:48:00Z" name: sequence-graph-d1134-69b44c49f9-tm66r.18a96946bb39bb67 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 1a8d3fd5-6c02-43b4-804f-2fe41b8e4f35 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:00Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d1134-69b44c49f9-tm66r namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: c0483f79-b854-42da-a3b9-ac80a7066269 kind: Event lastTimestamp: "2026-04-24T21:48:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-d1134-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:00Z" name: sequence-graph-d1134-69b44c49f9-tm66r.18a96946c33739b3 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 5e660c12-357e-460a-8ff5-5e46367689a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:01Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d1134-69b44c49f9-tm66r namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: c0483f79-b854-42da-a3b9-ac80a7066269 kind: Event lastTimestamp: "2026-04-24T21:48:01Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:48:01Z" name: sequence-graph-d1134-69b44c49f9-tm66r.18a96946f8369993 namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: 9447190d-088f-4b28-9e7e-80d2c9082f1c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d1134} kind: Pod name: sequence-graph-d1134-69b44c49f9-tm66r namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: c0483f79-b854-42da-a3b9-ac80a7066269 kind: Event lastTimestamp: "2026-04-24T21:48:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:01Z" name: sequence-graph-d1134-69b44c49f9-tm66r.18a96946f985d869 namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 0612956b-acda-4a55-b7e0-4d8d223bbc8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d1134} kind: Pod name: sequence-graph-d1134-69b44c49f9-tm66r namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: c0483f79-b854-42da-a3b9-ac80a7066269 kind: Event lastTimestamp: "2026-04-24T21:48:01Z" message: 'Created container: sequence-graph-d1134' metadata: creationTimestamp: "2026-04-24T21:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:01Z" name: sequence-graph-d1134-69b44c49f9-tm66r.18a96946fd2db883 namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcbceaed-537b-4d62-92b2-bdc43d49a2d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d1134} kind: Pod name: sequence-graph-d1134-69b44c49f9-tm66r namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: c0483f79-b854-42da-a3b9-ac80a7066269 kind: Event lastTimestamp: "2026-04-24T21:48:01Z" message: Started container sequence-graph-d1134 metadata: creationTimestamp: "2026-04-24T21:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:01Z" name: sequence-graph-d1134-69b44c49f9-tm66r.18a96946fda848c8 namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 024aeb8c-594f-478b-a43a-8b80ce5d4043 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d1134} kind: Pod name: sequence-graph-d1134-69b44c49f9-tm66r namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: c0483f79-b854-42da-a3b9-ac80a7066269 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Stopping container sequence-graph-d1134 metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: sequence-graph-d1134-69b44c49f9-tm66r.18a969b9e581edab namespace: kserve-ci-e2e-test resourceVersion: "24150" uid: 6c2f1d71-f392-4a84-a56e-c6784139a924 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d1134} kind: Pod name: sequence-graph-d1134-69b44c49f9-tm66r namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: c0483f79-b854-42da-a3b9-ac80a7066269 kind: Event lastTimestamp: "2026-04-24T21:56:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:43Z" name: sequence-graph-d1134-69b44c49f9-tm66r.18a969baa56e78ff namespace: kserve-ci-e2e-test resourceVersion: "24559" uid: 587e66f0-28f6-4f0c-8b50-e0c88203be99 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-d1134-69b44c49f9 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: ec1707e2-623b-4758-89e7-c785ab982859 kind: Event lastTimestamp: "2026-04-24T21:48:00Z" message: 'Created pod: sequence-graph-d1134-69b44c49f9-tm66r' metadata: creationTimestamp: "2026-04-24T21:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:00Z" name: sequence-graph-d1134-69b44c49f9.18a96946bae19ba8 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 9c1862e8-431c-43e5-9e35-243d91e238a2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-d1134 namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: 6fd0d54b-c194-429d-a823-2abfa39420a4 kind: Event lastTimestamp: "2026-04-24T21:48:00Z" message: Scaled up replica set sequence-graph-d1134-69b44c49f9 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:00Z" name: sequence-graph-d1134.18a96946ba0d2f4b namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 7b99f0b5-f8b1-403a-b698-45988c7bc7c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d1134 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 9ad204fe-0ef9-4656-9232-1914169f84d7 kind: Event lastTimestamp: "2026-04-24T21:48:00Z" message: InferenceGraph [sequence-graph-d1134] is Ready metadata: creationTimestamp: "2026-04-24T21:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:00Z" name: sequence-graph-d1134.18a96946bfa6e636 namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: f9cf8c2e-b0b1-4a6b-823f-3958087921b6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d1134 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 5a8b6260-ba28-4d17-b81b-7ffb0b99abaa kind: Event lastTimestamp: "2026-04-24T21:48:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:45Z" name: sequence-graph-d1134.18a9694a3915db1c namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: 0e2adf1a-7d20-489e-aed4-c9a8cdd7e71e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d1134 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 5a8b6260-ba28-4d17-b81b-7ffb0b99abaa kind: Event lastTimestamp: "2026-04-24T21:48:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:45Z" name: sequence-graph-d1134.18a9694a3964b161 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: 1bda28f9-36d9-413f-b9c1-6d0ab0148f67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:56:35.627753Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6842b-55bb6cfccf-g8lvd namespace: kserve-ci-e2e-test resourceVersion: "24474" uid: e3da9271-340b-4dd7-9eb3-9bbb1c51ec42 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-6842b-55bb6cfccf-g8lvd to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:56:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:56:35Z" name: splitter-graph-6842b-55bb6cfccf-g8lvd.18a969beac2ddb36 namespace: kserve-ci-e2e-test resourceVersion: "24480" uid: 7c0b5a41-edfc-40dd-8b87-00f817f06ed0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:35Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6842b-55bb6cfccf-g8lvd namespace: kserve-ci-e2e-test resourceVersion: "24477" uid: e3da9271-340b-4dd7-9eb3-9bbb1c51ec42 kind: Event lastTimestamp: "2026-04-24T21:56:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-6842b-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:35Z" name: splitter-graph-6842b-55bb6cfccf-g8lvd.18a969beb3d91221 namespace: kserve-ci-e2e-test resourceVersion: "24491" uid: 8ed0e447-7ae2-4ef8-a597-a0a8647ee6ba reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:36Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6842b-55bb6cfccf-g8lvd namespace: kserve-ci-e2e-test resourceVersion: "24484" uid: e3da9271-340b-4dd7-9eb3-9bbb1c51ec42 kind: Event lastTimestamp: "2026-04-24T21:56:36Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:56:36Z" name: splitter-graph-6842b-55bb6cfccf-g8lvd.18a969bee9225d13 namespace: kserve-ci-e2e-test resourceVersion: "24501" uid: 464eae6f-7d8d-4d0a-bc9a-e1d193aadbf7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6842b} kind: Pod name: splitter-graph-6842b-55bb6cfccf-g8lvd namespace: kserve-ci-e2e-test resourceVersion: "24477" uid: e3da9271-340b-4dd7-9eb3-9bbb1c51ec42 kind: Event lastTimestamp: "2026-04-24T21:56:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:36Z" name: splitter-graph-6842b-55bb6cfccf-g8lvd.18a969beea71ce5f namespace: kserve-ci-e2e-test resourceVersion: "24503" uid: 1b0a1361-e6df-4960-b35e-0e64fa59ccf5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6842b} kind: Pod name: splitter-graph-6842b-55bb6cfccf-g8lvd namespace: kserve-ci-e2e-test resourceVersion: "24477" uid: e3da9271-340b-4dd7-9eb3-9bbb1c51ec42 kind: Event lastTimestamp: "2026-04-24T21:56:36Z" message: 'Created container: splitter-graph-6842b' metadata: creationTimestamp: "2026-04-24T21:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:36Z" name: splitter-graph-6842b-55bb6cfccf-g8lvd.18a969beed8d73ec namespace: kserve-ci-e2e-test resourceVersion: "24505" uid: 68219319-95bd-4b34-b61b-3320afbb6dbe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6842b} kind: Pod name: splitter-graph-6842b-55bb6cfccf-g8lvd namespace: kserve-ci-e2e-test resourceVersion: "24477" uid: e3da9271-340b-4dd7-9eb3-9bbb1c51ec42 kind: Event lastTimestamp: "2026-04-24T21:56:36Z" message: Started container splitter-graph-6842b metadata: creationTimestamp: "2026-04-24T21:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:36Z" name: splitter-graph-6842b-55bb6cfccf-g8lvd.18a969beee01afbb namespace: kserve-ci-e2e-test resourceVersion: "24506" uid: af7ac1eb-3f6b-4743-bbff-6a93d80b5d10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6842b} kind: Pod name: splitter-graph-6842b-55bb6cfccf-g8lvd namespace: kserve-ci-e2e-test resourceVersion: "24477" uid: e3da9271-340b-4dd7-9eb3-9bbb1c51ec42 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: Stopping container splitter-graph-6842b metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:45Z" name: splitter-graph-6842b-55bb6cfccf-g8lvd.18a969c104fe0221 namespace: kserve-ci-e2e-test resourceVersion: "24598" uid: f35b3824-9839-40be-b667-78f86876c308 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6842b} kind: Pod name: splitter-graph-6842b-55bb6cfccf-g8lvd namespace: kserve-ci-e2e-test resourceVersion: "24477" uid: e3da9271-340b-4dd7-9eb3-9bbb1c51ec42 kind: Event lastTimestamp: "2026-04-24T21:57:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:12Z" name: splitter-graph-6842b-55bb6cfccf-g8lvd.18a969c1832fbc26 namespace: kserve-ci-e2e-test resourceVersion: "24954" uid: 06859d37-d5dd-4ccb-996d-a5f57953fd66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-6842b-55bb6cfccf namespace: kserve-ci-e2e-test resourceVersion: "24469" uid: 92c9d002-cbbe-4e9e-96a0-f7ba5a7c005e kind: Event lastTimestamp: "2026-04-24T21:56:35Z" message: 'Created pod: splitter-graph-6842b-55bb6cfccf-g8lvd' metadata: creationTimestamp: "2026-04-24T21:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:35Z" name: splitter-graph-6842b-55bb6cfccf.18a969beabb32be1 namespace: kserve-ci-e2e-test resourceVersion: "24476" uid: 8fff6094-7e89-4ff3-aa80-19f44d1fc388 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-6842b namespace: kserve-ci-e2e-test resourceVersion: "24467" uid: 1af0b7d0-936b-41a1-a650-77c1ca446621 kind: Event lastTimestamp: "2026-04-24T21:56:35Z" message: Scaled up replica set splitter-graph-6842b-55bb6cfccf from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:35Z" name: splitter-graph-6842b.18a969beaae1bed4 namespace: kserve-ci-e2e-test resourceVersion: "24472" uid: a08224a4-4806-413a-83ca-dbfeb4e91373 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6842b namespace: kserve-ci-e2e-test resourceVersion: "24486" uid: 009af226-82f0-4c60-bc8f-19baf6b28e2f kind: Event lastTimestamp: "2026-04-24T21:56:35Z" message: InferenceGraph [splitter-graph-6842b] is Ready metadata: creationTimestamp: "2026-04-24T21:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:35Z" name: splitter-graph-6842b.18a969beb1da9bda namespace: kserve-ci-e2e-test resourceVersion: "24488" uid: 590f075f-2efb-4e87-b895-85944a40e718 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:57:45.950634Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6941b-78cf747fb6-bxvdp namespace: kserve-ci-e2e-test resourceVersion: "25288" uid: 783d3997-6bfd-4ec9-bff5-91a0abf8f875 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-6941b-78cf747fb6-bxvdp to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:57:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:57:45Z" name: splitter-graph-6941b-78cf747fb6-bxvdp.18a969cf0bbfdb2d namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: b197ca73-6b07-4169-b30d-58da5c02f8dc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:46Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6941b-78cf747fb6-bxvdp namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: 783d3997-6bfd-4ec9-bff5-91a0abf8f875 kind: Event lastTimestamp: "2026-04-24T21:57:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-6941b-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:46Z" name: splitter-graph-6941b-78cf747fb6-bxvdp.18a969cf1563fbf9 namespace: kserve-ci-e2e-test resourceVersion: "25306" uid: 65f21e2d-bfec-4c52-8c4c-8435aa24c365 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:46Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6941b-78cf747fb6-bxvdp namespace: kserve-ci-e2e-test resourceVersion: "25296" uid: 783d3997-6bfd-4ec9-bff5-91a0abf8f875 kind: Event lastTimestamp: "2026-04-24T21:57:46Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:57:46Z" name: splitter-graph-6941b-78cf747fb6-bxvdp.18a969cf48e959f3 namespace: kserve-ci-e2e-test resourceVersion: "25316" uid: 200352c3-f367-4875-8d07-2cc01cf001a7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6941b} kind: Pod name: splitter-graph-6941b-78cf747fb6-bxvdp namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: 783d3997-6bfd-4ec9-bff5-91a0abf8f875 kind: Event lastTimestamp: "2026-04-24T21:57:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:47Z" name: splitter-graph-6941b-78cf747fb6-bxvdp.18a969cf4a392729 namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: 8be4b3f8-e762-41b8-be07-f7a160158fb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6941b} kind: Pod name: splitter-graph-6941b-78cf747fb6-bxvdp namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: 783d3997-6bfd-4ec9-bff5-91a0abf8f875 kind: Event lastTimestamp: "2026-04-24T21:57:47Z" message: 'Created container: splitter-graph-6941b' metadata: creationTimestamp: "2026-04-24T21:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:47Z" name: splitter-graph-6941b-78cf747fb6-bxvdp.18a969cf4d6edf75 namespace: kserve-ci-e2e-test resourceVersion: "25319" uid: ca0da683-6582-488c-b9ba-0dd0b10e1d62 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6941b} kind: Pod name: splitter-graph-6941b-78cf747fb6-bxvdp namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: 783d3997-6bfd-4ec9-bff5-91a0abf8f875 kind: Event lastTimestamp: "2026-04-24T21:57:47Z" message: Started container splitter-graph-6941b metadata: creationTimestamp: "2026-04-24T21:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:47Z" name: splitter-graph-6941b-78cf747fb6-bxvdp.18a969cf4de78eda namespace: kserve-ci-e2e-test resourceVersion: "25320" uid: 29fc4061-4a57-4885-b050-df2b7d3f9225 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6941b} kind: Pod name: splitter-graph-6941b-78cf747fb6-bxvdp namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: 783d3997-6bfd-4ec9-bff5-91a0abf8f875 kind: Event lastTimestamp: "2026-04-24T22:06:00Z" message: Stopping container splitter-graph-6941b metadata: creationTimestamp: "2026-04-24T22:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:06:00Z" name: splitter-graph-6941b-78cf747fb6-bxvdp.18a96a422f47f270 namespace: kserve-ci-e2e-test resourceVersion: "29025" uid: 9527dccb-5102-42ee-b270-30671b74ab6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6941b} kind: Pod name: splitter-graph-6941b-78cf747fb6-bxvdp namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: 783d3997-6bfd-4ec9-bff5-91a0abf8f875 kind: Event lastTimestamp: "2026-04-24T22:06:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:06:29Z" name: splitter-graph-6941b-78cf747fb6-bxvdp.18a96a43041bd68d namespace: kserve-ci-e2e-test resourceVersion: "29263" uid: 903edba5-079f-4108-981a-025564c502b0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-6941b-78cf747fb6 namespace: kserve-ci-e2e-test resourceVersion: "25285" uid: 542855e0-8733-49a5-b006-863d5a76d0d6 kind: Event lastTimestamp: "2026-04-24T21:57:45Z" message: 'Created pod: splitter-graph-6941b-78cf747fb6-bxvdp' metadata: creationTimestamp: "2026-04-24T21:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:45Z" name: splitter-graph-6941b-78cf747fb6.18a969cf0b18d51e namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: 39cff68a-ee28-42da-81c9-2edce60a4c38 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: f99c25f1-86da-4090-b770-7ba16260fa4b kind: Event lastTimestamp: "2026-04-24T21:57:45Z" message: Scaled up replica set splitter-graph-6941b-78cf747fb6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:45Z" name: splitter-graph-6941b.18a969cf0a248345 namespace: kserve-ci-e2e-test resourceVersion: "25286" uid: 85792d89-fddb-41de-80e1-15011c5907d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "25301" uid: 553d1ff3-224d-4bbc-95ef-39e1cdc76f9c kind: Event lastTimestamp: "2026-04-24T21:57:46Z" message: InferenceGraph [splitter-graph-6941b] is Ready metadata: creationTimestamp: "2026-04-24T21:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:46Z" name: splitter-graph-6941b.18a969cf104ddb70 namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 84593cc5-f388-4390-9a12-6c38a9bbda9c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "25273" uid: 553d1ff3-224d-4bbc-95ef-39e1cdc76f9c kind: Event lastTimestamp: "2026-04-24T21:57:46Z" message: 'Failed to update status for InferenceGraph "splitter-graph-6941b": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-6941b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:46Z" name: splitter-graph-6941b.18a969cf132e0758 namespace: kserve-ci-e2e-test resourceVersion: "25304" uid: 9f65bdfa-b8fb-467a-8798-e1e27f02d32e reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "25273" uid: 553d1ff3-224d-4bbc-95ef-39e1cdc76f9c kind: Event lastTimestamp: "2026-04-24T21:57:46Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-6941b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:46Z" name: splitter-graph-6941b.18a969cf132e4a6b namespace: kserve-ci-e2e-test resourceVersion: "25305" uid: e2e2a0dd-70df-46c3-978a-6990d96c7e1b reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "25284" uid: a23c8716-c617-4bda-9113-b635658f3ca4 kind: Event lastTimestamp: "2026-04-24T21:58:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:00Z" name: splitter-graph-6941b.18a969d2889a08ac namespace: kserve-ci-e2e-test resourceVersion: "25428" uid: 8ff455ef-d0ef-4660-a754-9a52f0015de9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "25284" uid: a23c8716-c617-4bda-9113-b635658f3ca4 kind: Event lastTimestamp: "2026-04-24T21:58:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:00Z" name: splitter-graph-6941b.18a969d2891e3ffb namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 11a7f5c0-2ad3-42a4-abd9-27a7c8f01fcf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "25429" uid: a23c8716-c617-4bda-9113-b635658f3ca4 kind: Event lastTimestamp: "2026-04-24T21:58:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:15Z" name: splitter-graph-6941b.18a969d607dc5533 namespace: kserve-ci-e2e-test resourceVersion: "25544" uid: d03310ea-2805-4fcb-ab31-b27aa86fec8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "25429" uid: a23c8716-c617-4bda-9113-b635658f3ca4 kind: Event lastTimestamp: "2026-04-24T21:58:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:15Z" name: splitter-graph-6941b.18a969d60857becf namespace: kserve-ci-e2e-test resourceVersion: "25546" uid: 34d407b1-f91f-413d-9b14-3749d73a4ff8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:36:35.384996Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-54155-predictor-744c89d589-bgsvq to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:36:35Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a73822c789 namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 9ddc7a2d-a9ea-444d-a5b5-1fe5eec950ac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-54155-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:35Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a7456717ca namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: a35f88a9-9090-4e8e-ac69-2ea56fe53a98 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:36:36Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a774c98d9a namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: d8572977-de84-402a-9427-9e06e1d6079c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a776445b72 namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: bf251eb8-be63-47e0-8369-3e3690599604 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a77a42920e namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 31f78be5-ec59-427a-97e1-cccbbf129291 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a77ac5a791 namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: 3b5aadd9-cda7-496b-a496-cc2ba056336e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a77ad07261 namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: ca6c2eab-2de0-480f-98cd-390ded1cbfba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a77dd630f6 namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 01090151-f3f8-4af8-a143-830ad9759f6e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:36:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:36Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a77e4622f3 namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: e068b43d-9c09-4356-bff2-8647c3658937 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:45:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:53Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a968a7a7eccf67 namespace: kserve-ci-e2e-test resourceVersion: "19021" uid: ef336324-2c9e-4481-a1c0-cb2751195b51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a9692865746862 namespace: kserve-ci-e2e-test resourceVersion: "18928" uid: e094f281-a9f3-4929-9ece-d879b26b1a69 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a969286574daf0 namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: d5777e36-18d4-4e89-94fd-5f5831e6761b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-54155-predictor-744c89d589-bgsvq namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: a38b40de-e304-4264-9768-c8b049b630a2 kind: Event lastTimestamp: "2026-04-24T21:45:53Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:53Z" name: success-200-isvc-54155-predictor-744c89d589-bgsvq.18a969291c412182 namespace: kserve-ci-e2e-test resourceVersion: "19020" uid: d9a5be1f-d600-48b1-bad4-032db80a53a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-54155-predictor-744c89d589 namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: ebd22cb0-8b3b-40b7-9b1a-c88d3b4b0e84 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: 'Created pod: success-200-isvc-54155-predictor-744c89d589-bgsvq' metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:35Z" name: success-200-isvc-54155-predictor-744c89d589.18a968a735bd9f46 namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: 039a9f74-c401-482c-8f56-6c1ec72dcbba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: dc84384b-3fc7-478d-9c05-7677ebeb4338 kind: Event lastTimestamp: "2026-04-24T21:36:35Z" message: Scaled up replica set success-200-isvc-54155-predictor-744c89d589 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:35Z" name: success-200-isvc-54155-predictor.18a968a73146d7df namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: 3c8d2dca-a244-4bfe-98e2-350ffbab9567 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: bafbf55f-9381-454b-929f-f1cd7801a935 kind: Event lastTimestamp: "2026-04-24T21:36:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:50Z" name: success-200-isvc-54155-predictor.18a968aab0772ac0 namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: b7e5831c-60c9-48cb-928a-050cecf09ae5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: bafbf55f-9381-454b-929f-f1cd7801a935 kind: Event lastTimestamp: "2026-04-24T21:36:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:50Z" name: success-200-isvc-54155-predictor.18a968aab0d0c0a6 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: d6efa1b9-d520-44b1-b0e3-9715f1714f4d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: bafbf55f-9381-454b-929f-f1cd7801a935 kind: Event lastTimestamp: "2026-04-24T21:37:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:50Z" name: success-200-isvc-54155-predictor.18a968ae2f91aab0 namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 884d669f-5015-4f2d-bf6a-feed5068e7ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54155-predictor namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: bafbf55f-9381-454b-929f-f1cd7801a935 kind: Event lastTimestamp: "2026-04-24T21:37:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:50Z" name: success-200-isvc-54155-predictor.18a968ae2fff9f86 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: fe194efe-96d0-4025-b1e6-e82360dacd4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:36:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-54155 namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: 3b4849a7-abc3-4dfc-9430-feed808ae28a kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:23Z" name: success-200-isvc-54155.18a968a73114c8cc namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 51822f66-5b03-462a-b6df-d0f111f3edcb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-54155 namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: 3b4849a7-abc3-4dfc-9430-feed808ae28a kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: InferenceService [success-200-isvc-54155] is Ready metadata: creationTimestamp: "2026-04-24T21:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:37:23Z" name: success-200-isvc-54155.18a968b2635c80be namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 57c30b3f-5c69-40cd-a89a-75a7e3d755d4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:45:50.209636Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6445b-predictor-769c496d67-hc2tf to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:45:50Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928664304a3 namespace: kserve-ci-e2e-test resourceVersion: "18934" uid: 1264ac8a-d3ef-49b2-a965-c7bd09e2fa4f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6445b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:50Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928764a9e59 namespace: kserve-ci-e2e-test resourceVersion: "18969" uid: f6f41b2c-bc48-4cf5-a83c-a66e339245c4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18940" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:45:51Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:45:51Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928af8781f5 namespace: kserve-ci-e2e-test resourceVersion: "18993" uid: 2409c287-a3d9-4ef9-b8c9-4d7bc4aa54e7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:45:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:51Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928b10474ef namespace: kserve-ci-e2e-test resourceVersion: "18995" uid: 49c4596a-f0b0-4521-ae9f-1f28821e75ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:45:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:51Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928b4014adf namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: eacae19b-ac40-48bf-80fe-40e48fec277d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:45:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:51Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928b48c7b49 namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 943a5a43-f582-4c41-9fdd-1906e41f7f42 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:45:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:51Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928b4985325 namespace: kserve-ci-e2e-test resourceVersion: "18999" uid: ed53e6f1-caa5-425d-aeb1-7925ca618bd8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:45:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:51Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928b75fbd8b namespace: kserve-ci-e2e-test resourceVersion: "19000" uid: c41c6aff-b267-4282-9782-addcf8fda4b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:45:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:51Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928b7ce3592 namespace: kserve-ci-e2e-test resourceVersion: "19001" uid: 32f3686f-4ed2-4e93-ad51-c34c64bf8025 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:46:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:27Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96928cb2650b1 namespace: kserve-ci-e2e-test resourceVersion: "19514" uid: cdd858a3-395e-4113-929d-162a9cb877cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96938c092a142 namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: 18168725-4c04-43b4-a272-b7ce1c8ac889 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a96938c092e297 namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: de398a7b-e99a-4786-805a-2feca9c4923f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6445b-predictor-769c496d67-hc2tf namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 9c0e3ab9-32b9-4155-ad77-6da651f96cf3 kind: Event lastTimestamp: "2026-04-24T21:47:02Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:02Z" name: success-200-isvc-6445b-predictor-769c496d67-hc2tf.18a9693953426101 namespace: kserve-ci-e2e-test resourceVersion: "20000" uid: e862fa61-7dee-4f9d-b21d-ba2ae1cd563f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6445b-predictor-769c496d67 namespace: kserve-ci-e2e-test resourceVersion: "18909" uid: 3dc7e030-d1f4-4bc8-9864-6c64f5cfa2fd kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: 'Created pod: success-200-isvc-6445b-predictor-769c496d67-hc2tf' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:50Z" name: success-200-isvc-6445b-predictor-769c496d67.18a9692865b63186 namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: ce5a2921-4ea0-4910-9f6e-f1119bf4680f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18903" uid: 5e1e279f-e81b-43c8-9151-bc50789bf697 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: Scaled up replica set success-200-isvc-6445b-predictor-769c496d67 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:50Z" name: success-200-isvc-6445b-predictor.18a969286377cd86 namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: d68ff3d8-5ad7-4c42-aad0-609bc5e4fb05 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18907" uid: c39e8c44-8116-4f67-961e-3b6833bef7e1 kind: Event lastTimestamp: "2026-04-24T21:46:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:35Z" name: success-200-isvc-6445b-predictor.18a9692be204f227 namespace: kserve-ci-e2e-test resourceVersion: "19574" uid: ef43c550-befa-47e6-8ac3-f02f406d9242 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18907" uid: c39e8c44-8116-4f67-961e-3b6833bef7e1 kind: Event lastTimestamp: "2026-04-24T21:46:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:35Z" name: success-200-isvc-6445b-predictor.18a9692be26e5007 namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: f59b87d1-c5cb-4bf2-ac29-8911747b4e72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19142" uid: c39e8c44-8116-4f67-961e-3b6833bef7e1 kind: Event lastTimestamp: "2026-04-24T21:46:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:50Z" name: success-200-isvc-6445b-predictor.18a9692f61b9ebe4 namespace: kserve-ci-e2e-test resourceVersion: "19729" uid: 99634566-3b79-4222-9a5b-5e625d25c911 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6445b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19142" uid: c39e8c44-8116-4f67-961e-3b6833bef7e1 kind: Event lastTimestamp: "2026-04-24T21:46:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:50Z" name: success-200-isvc-6445b-predictor.18a9692f6206b009 namespace: kserve-ci-e2e-test resourceVersion: "19730" uid: 1a3e6000-f012-4be4-9305-968c0b5c6d43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6445b namespace: kserve-ci-e2e-test resourceVersion: "18879" uid: f41126f5-e0ab-4c73-840e-90f0cd94b1a5 kind: Event lastTimestamp: "2026-04-24T21:46:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:38Z" name: success-200-isvc-6445b.18a969286353ec99 namespace: kserve-ci-e2e-test resourceVersion: "19623" uid: 3a370426-f82b-415a-9bcb-d7338d8f67e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6445b namespace: kserve-ci-e2e-test resourceVersion: "18879" uid: f41126f5-e0ab-4c73-840e-90f0cd94b1a5 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-6445b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6445b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:50Z" name: success-200-isvc-6445b.18a969286429d9ae namespace: kserve-ci-e2e-test resourceVersion: "18922" uid: b6bf7fb6-3e73-4537-9f25-3b82f88fa7ef reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6445b namespace: kserve-ci-e2e-test resourceVersion: "18879" uid: f41126f5-e0ab-4c73-840e-90f0cd94b1a5 kind: Event lastTimestamp: "2026-04-24T21:45:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6445b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:50Z" name: success-200-isvc-6445b.18a96928642a244f namespace: kserve-ci-e2e-test resourceVersion: "18924" uid: 046fd432-8fba-48fe-b060-0defcd7c7178 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6445b namespace: kserve-ci-e2e-test resourceVersion: "19620" uid: f41126f5-e0ab-4c73-840e-90f0cd94b1a5 kind: Event lastTimestamp: "2026-04-24T21:46:38Z" message: InferenceService [success-200-isvc-6445b] is Ready metadata: creationTimestamp: "2026-04-24T21:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:38Z" name: success-200-isvc-6445b.18a9693386e9678b namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: 2dac5923-8135-4d0c-bd29-4b0dbc307d83 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:55:35.616807Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23745" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:55:35Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969b0b33f8810 namespace: kserve-ci-e2e-test resourceVersion: "23760" uid: d1c3a845-65e3-487c-9bb3-dd689c9168e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23774" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:55:36Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969b0cc501d3c namespace: kserve-ci-e2e-test resourceVersion: "23801" uid: a04a3eb0-3f34-4d0c-8f7e-536ee5b5a0b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969b0ce0345d5 namespace: kserve-ci-e2e-test resourceVersion: "23803" uid: 008c27bc-03b1-4ff0-b49a-ffeab8b083b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969b0d281d692 namespace: kserve-ci-e2e-test resourceVersion: "23806" uid: f93332ae-3018-43b6-8e93-7793b0937a19 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969b0d3013831 namespace: kserve-ci-e2e-test resourceVersion: "23808" uid: 9ffabe33-0d2e-4009-bbee-c62d072be955 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969b0d30ad86f namespace: kserve-ci-e2e-test resourceVersion: "23809" uid: 01fef697-8fb9-4e23-b8e5-36835be240b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969b0d5a2074b namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: 6f2d6069-c730-428e-af60-82a470457053 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:55:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:36Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969b0d61144cf namespace: kserve-ci-e2e-test resourceVersion: "23814" uid: 7f6bf230-57f0-4db1-9cc9-9cef73827e81 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:56:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:12Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969b0f22335df namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 5bbf5613-b39b-42cb-a558-226f5dc4420d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:45Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969c10c60227b namespace: kserve-ci-e2e-test resourceVersion: "24639" uid: efea9769-1055-44b0-9314-22c90e68da1e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:45Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969c10c60a901 namespace: kserve-ci-e2e-test resourceVersion: "24643" uid: 79138ed3-9b0d-48c5-9927-36113f8fb450 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc namespace: kserve-ci-e2e-test resourceVersion: "23752" uid: 9302faf7-7f03-465f-8e33-8ea9cdf22fdd kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc.18a969c17a6f8127 namespace: kserve-ci-e2e-test resourceVersion: "24722" uid: cb17f8b0-3182-4bfa-94f4-e6c43370fc1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6842b-predictor-7bd4c8d8df namespace: kserve-ci-e2e-test resourceVersion: "23732" uid: 1fbed400-6d1b-40fd-b926-9e65ed8324c1 kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: 'Created pod: success-200-isvc-6842b-predictor-7bd4c8d8df-c8zpc' metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:55:35Z" name: success-200-isvc-6842b-predictor-7bd4c8d8df.18a969b0b2683981 namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 1c24d8af-dd9b-44a5-859d-e9952e3b4f44 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23725" uid: d6fa103d-e796-4fe8-bd1f-2173c0793ec8 kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: Scaled up replica set success-200-isvc-6842b-predictor-7bd4c8d8df from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:55:35Z" name: success-200-isvc-6842b-predictor.18a969b0ae17e2c8 namespace: kserve-ci-e2e-test resourceVersion: "23735" uid: 35b52fce-43ae-41f7-a582-05dd612a4be6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:55:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23731" uid: 0e656acf-7d0c-41cd-b0d0-6fdc426a42f8 kind: Event lastTimestamp: "2026-04-24T21:56:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:20Z" name: success-200-isvc-6842b-predictor.18a969b42d8097d5 namespace: kserve-ci-e2e-test resourceVersion: "24317" uid: d640bec4-4f26-489b-a473-1a08f09b149b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:55:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23731" uid: 0e656acf-7d0c-41cd-b0d0-6fdc426a42f8 kind: Event lastTimestamp: "2026-04-24T21:56:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:20Z" name: success-200-isvc-6842b-predictor.18a969b42e22f137 namespace: kserve-ci-e2e-test resourceVersion: "24318" uid: 4cc222e7-d385-4ec0-ae4f-621a58b53f5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23944" uid: 0e656acf-7d0c-41cd-b0d0-6fdc426a42f8 kind: Event lastTimestamp: "2026-04-24T21:56:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:35Z" name: success-200-isvc-6842b-predictor.18a969beaabedd13 namespace: kserve-ci-e2e-test resourceVersion: "24471" uid: 4dd88767-ca82-4d88-aa96-25dea1178cfa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6842b-predictor namespace: kserve-ci-e2e-test resourceVersion: "23944" uid: 0e656acf-7d0c-41cd-b0d0-6fdc426a42f8 kind: Event lastTimestamp: "2026-04-24T21:56:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:35Z" name: success-200-isvc-6842b-predictor.18a969beab354468 namespace: kserve-ci-e2e-test resourceVersion: "24475" uid: a0962812-107b-4432-93a2-59a76d78c2d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6842b namespace: kserve-ci-e2e-test resourceVersion: "23710" uid: 4739174c-3a81-4e88-ab06-705f77c79c3b kind: Event lastTimestamp: "2026-04-24T21:56:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:22Z" name: success-200-isvc-6842b.18a969b0adf10d71 namespace: kserve-ci-e2e-test resourceVersion: "24343" uid: 64d6ba70-3a4d-4cab-8cee-f0aa23e01a6f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6842b namespace: kserve-ci-e2e-test resourceVersion: "23710" uid: 4739174c-3a81-4e88-ab06-705f77c79c3b kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: 'Failed to update status for InferenceService "success-200-isvc-6842b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6842b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:55:35Z" name: success-200-isvc-6842b.18a969b0aecc5cc6 namespace: kserve-ci-e2e-test resourceVersion: "23738" uid: a7db1eee-016b-4338-af96-a33bb8cf4578 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6842b namespace: kserve-ci-e2e-test resourceVersion: "23710" uid: 4739174c-3a81-4e88-ab06-705f77c79c3b kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6842b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:55:35Z" name: success-200-isvc-6842b.18a969b0aeccc30c namespace: kserve-ci-e2e-test resourceVersion: "23741" uid: 37d94343-a519-4e57-89e9-e56822296048 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6842b namespace: kserve-ci-e2e-test resourceVersion: "24340" uid: 4739174c-3a81-4e88-ab06-705f77c79c3b kind: Event lastTimestamp: "2026-04-24T21:56:22Z" message: InferenceService [success-200-isvc-6842b] is Ready metadata: creationTimestamp: "2026-04-24T21:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:22Z" name: success-200-isvc-6842b.18a969bbae07b514 namespace: kserve-ci-e2e-test resourceVersion: "24341" uid: 011953b0-abd2-4b45-a998-6f875a3dbd9b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:56:45.877087Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24649" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:56:45Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c10f164428 namespace: kserve-ci-e2e-test resourceVersion: "24654" uid: 088de598-4cc6-4fb4-aa33-03093689081f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T21:56:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6941b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:46Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c11d269a4c namespace: kserve-ci-e2e-test resourceVersion: "24685" uid: 1e9dbed8-e1ef-4407-9311-e3d8be828121 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24660" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T21:56:46Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:56:46Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c14c20d39b namespace: kserve-ci-e2e-test resourceVersion: "24700" uid: 7f6a3131-db0d-4cd1-b573-e78c1bf56405 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T21:56:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:46Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c14db647c1 namespace: kserve-ci-e2e-test resourceVersion: "24703" uid: 016bc13a-2128-44cb-9f7b-83a29010059b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c15213087e namespace: kserve-ci-e2e-test resourceVersion: "24705" uid: b05369af-fd1a-45cc-bd32-5d5c0664fc8c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c1528c4ee7 namespace: kserve-ci-e2e-test resourceVersion: "24706" uid: c18a6c7c-d481-4e92-af2d-8b2954440d27 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c152994c48 namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: eec2c333-8c22-47bf-9833-dcdc346f82c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c1563f517f namespace: kserve-ci-e2e-test resourceVersion: "24711" uid: f5f647b6-e622-47d0-bbf8-5f06cc7f3fa8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T21:56:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:47Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c156b691f0 namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 02ca464a-9ef1-4132-aa7d-be09cd850029 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T21:57:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:24Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a969c1c6229d50 namespace: kserve-ci-e2e-test resourceVersion: "25096" uid: 12f3b608-bfe4-4f13-a88c-4cf29dec10d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T22:06:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:06:00Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a96a4234e27e3c namespace: kserve-ci-e2e-test resourceVersion: "29039" uid: 357b79c3-688a-4ba3-9068-1a437a1311c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 0e9e117e-b849-44a2-b9c2-e2290e170fe1 kind: Event lastTimestamp: "2026-04-24T22:06:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:06:00Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf.18a96a4234e30758 namespace: kserve-ci-e2e-test resourceVersion: "29041" uid: b84724d0-d678-4b21-9942-0c9d9ace5857 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6941b-predictor-58bfbfdd8f namespace: kserve-ci-e2e-test resourceVersion: "24638" uid: d652685b-88fa-4e2f-b25d-bfbef9471880 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: 'Created pod: success-200-isvc-6941b-predictor-58bfbfdd8f-nrzjf' metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:45Z" name: success-200-isvc-6941b-predictor-58bfbfdd8f.18a969c10ea729b8 namespace: kserve-ci-e2e-test resourceVersion: "24651" uid: b4550838-c29e-4cda-93d8-3120fc68a4f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24634" uid: 39870b89-dbc4-4271-ad68-41a5927b29a7 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: Scaled up replica set success-200-isvc-6941b-predictor-58bfbfdd8f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:45Z" name: success-200-isvc-6941b-predictor.18a969c10ca75a0a namespace: kserve-ci-e2e-test resourceVersion: "24641" uid: db8d3f1b-ee67-41d7-8b3a-953ae08d8322 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:57:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24637" uid: 276b518a-2c22-4dec-9e0a-3fb40f5fc03f kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:57:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:15Z" name: success-200-isvc-6941b-predictor.18a969c48b3282d3 namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 72b91b4a-b63a-46c5-b6ff-277620251451 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:57:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24637" uid: 276b518a-2c22-4dec-9e0a-3fb40f5fc03f kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:57:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:15Z" name: success-200-isvc-6941b-predictor.18a969c48ba0900b namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: ae943bf9-4f2c-46af-a948-9a6f03198798 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 276b518a-2c22-4dec-9e0a-3fb40f5fc03f kind: Event lastTimestamp: "2026-04-24T21:58:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:00Z" name: success-200-isvc-6941b-predictor.18a969cb893ff9b9 namespace: kserve-ci-e2e-test resourceVersion: "25426" uid: 01b0569b-e36a-4df3-96ae-deb917897dec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 276b518a-2c22-4dec-9e0a-3fb40f5fc03f kind: Event lastTimestamp: "2026-04-24T21:58:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:00Z" name: success-200-isvc-6941b-predictor.18a969cb89c84d6d namespace: kserve-ci-e2e-test resourceVersion: "25427" uid: 1272f2b9-8c6d-4538-87d4-88f5a3a7b6bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "24608" uid: bddb478b-b704-4d2e-bca5-f275f8249c03 kind: Event lastTimestamp: "2026-04-24T21:57:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:35Z" name: success-200-isvc-6941b.18a969c10c883a8e namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: 642c6e6b-306e-444d-8604-0aaaabd95c8f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "24608" uid: bddb478b-b704-4d2e-bca5-f275f8249c03 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-6941b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6941b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:45Z" name: success-200-isvc-6941b.18a969c10ce86446 namespace: kserve-ci-e2e-test resourceVersion: "24644" uid: 98c697b3-38e3-4a99-913d-4b470f99f4d6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "24608" uid: bddb478b-b704-4d2e-bca5-f275f8249c03 kind: Event lastTimestamp: "2026-04-24T21:56:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6941b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:45Z" name: success-200-isvc-6941b.18a969c10ce8ce29 namespace: kserve-ci-e2e-test resourceVersion: "24645" uid: c18b5d5d-681b-4e42-affe-bffb9f718a9d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "25185" uid: bddb478b-b704-4d2e-bca5-f275f8249c03 kind: Event lastTimestamp: "2026-04-24T21:57:35Z" message: InferenceService [success-200-isvc-6941b] is Ready metadata: creationTimestamp: "2026-04-24T21:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:35Z" name: success-200-isvc-6941b.18a969cc8309bdfc namespace: kserve-ci-e2e-test resourceVersion: "25186" uid: 49d08dab-136b-46d2-99f5-c8f987c30602 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:35:55.380335Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14057" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9689de7ac1bc8 namespace: kserve-ci-e2e-test resourceVersion: "14062" uid: b5937077-5ce5-4c66-a7ac-0072be797f59 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14067" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9689e00cef3a3 namespace: kserve-ci-e2e-test resourceVersion: "14103" uid: 1bbfb365-81fe-4751-8196-45bded2540eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9689e022cebfa namespace: kserve-ci-e2e-test resourceVersion: "14105" uid: c331f0a4-054d-4768-b9ec-7c165002ddf1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9689e05dde05f namespace: kserve-ci-e2e-test resourceVersion: "14106" uid: 143eb45e-20b0-44db-a723-54fd6c49af58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9689e06884783 namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: c110121e-73c5-4953-bcdd-efb2df8c5189 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9689e069a6162 namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: c12f692a-817a-468e-a82b-115a1e57358b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9689e0abd2462 namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: d792fc97-d915-45f8-a28a-f03dcfe4a37b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9689e0b3c40a1 namespace: kserve-ci-e2e-test resourceVersion: "14116" uid: f6ab7884-a7e9-415d-bf81-e4b359d31533 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:45:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:13Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9689e13d453d5 namespace: kserve-ci-e2e-test resourceVersion: "18566" uid: b8a34201-f2ac-4840-a305-5a75d6b44141 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9691f1b225478 namespace: kserve-ci-e2e-test resourceVersion: "18457" uid: a4ff9d05-b4d5-4d84-81bf-82188d7b0ca6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9691f1b239c57 namespace: kserve-ci-e2e-test resourceVersion: "18460" uid: d267ab00-774e-4cd4-94af-c9d5019e929a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 2fa5eddf-8404-4732-93a3-3f6f1e0b7d1d kind: Event lastTimestamp: "2026-04-24T21:45:13Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:13Z" name: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc.18a9691fc43bf147 namespace: kserve-ci-e2e-test resourceVersion: "18564" uid: 32349156-3702-4f3a-9037-6162d3e317b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6d3ba-predictor-5578786d5c namespace: kserve-ci-e2e-test resourceVersion: "14039" uid: 63c0d48b-4e07-4eb6-a59c-f25a822d28a7 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'Created pod: success-200-isvc-6d3ba-predictor-5578786d5c-2xfmc' metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor-5578786d5c.18a9689de750125c namespace: kserve-ci-e2e-test resourceVersion: "14061" uid: c9569bca-68fe-458f-b276-749fbb659cce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: ed2c879a-d300-4b6d-9ed6-4ba62f21eabd kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Scaled up replica set success-200-isvc-6d3ba-predictor-5578786d5c from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba-predictor.18a9689de41c3df7 namespace: kserve-ci-e2e-test resourceVersion: "14045" uid: 59eae339-7be9-465b-ac18-0452b76b70a8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:36:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 17acc987-f0c6-4ed9-8a01-a3b5d9a40401 kind: Event lastTimestamp: "2026-04-24T21:36:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:40Z" name: success-200-isvc-6d3ba-predictor.18a968a162ad476e namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 2775868c-775d-4dc1-989e-9aa56a9aa16a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:36:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 17acc987-f0c6-4ed9-8a01-a3b5d9a40401 kind: Event lastTimestamp: "2026-04-24T21:36:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:40Z" name: success-200-isvc-6d3ba-predictor.18a968a1632c94c0 namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: e0d47677-d060-46e9-9242-d0067cd08039 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 17acc987-f0c6-4ed9-8a01-a3b5d9a40401 kind: Event lastTimestamp: "2026-04-24T21:36:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:55Z" name: success-200-isvc-6d3ba-predictor.18a968abe04532d5 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: aaa01136-d9d3-41b5-9fa9-e3ed6ecb96f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d3ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 17acc987-f0c6-4ed9-8a01-a3b5d9a40401 kind: Event lastTimestamp: "2026-04-24T21:36:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:55Z" name: success-200-isvc-6d3ba-predictor.18a968abe1454d28 namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: de9ec5ec-13dc-4608-8f08-fead3efc6ec4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: 0293b514-29a3-4c2e-af3e-ba5ba593d14c kind: Event lastTimestamp: "2026-04-24T21:36:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:43Z" name: success-200-isvc-6d3ba.18a9689de3eff590 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 8b0a5cea-541f-4c8c-949b-940bff373586 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: 0293b514-29a3-4c2e-af3e-ba5ba593d14c kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'Failed to update status for InferenceService "success-200-isvc-6d3ba": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6d3ba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba.18a9689de5223458 namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: a08acde9-5133-4773-9298-5a22f5f23d24 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: 0293b514-29a3-4c2e-af3e-ba5ba593d14c kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6d3ba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-6d3ba.18a9689de5229b45 namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: 8e544972-33e7-4e9d-89a5-750a23031b15 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 0293b514-29a3-4c2e-af3e-ba5ba593d14c kind: Event lastTimestamp: "2026-04-24T21:36:43Z" message: InferenceService [success-200-isvc-6d3ba] is Ready metadata: creationTimestamp: "2026-04-24T21:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:43Z" name: success-200-isvc-6d3ba.18a968a90c65eca6 namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: 3350a0d6-7e7c-48d5-9985-cb67f6207838 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:34:24.904993Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12955" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:34:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:34:24Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a96888d6ebfd02 namespace: kserve-ci-e2e-test resourceVersion: "12959" uid: 0fffc2b0-ef3b-4411-86ab-610162e48304 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12967" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:34:25Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a96888efe2bc66 namespace: kserve-ci-e2e-test resourceVersion: "13030" uid: 8a2738df-eab3-45ba-9d4b-c56620a8cd24 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:34:25Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" metadata: creationTimestamp: "2026-04-24T21:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:25Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a96888f22deb46 namespace: kserve-ci-e2e-test resourceVersion: "13041" uid: 26407c71-91ea-4c1b-98f9-d76ee3b3218a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" in 17.051s (17.051s including waiting). Image size: 1335723119 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9688cea87a178 namespace: kserve-ci-e2e-test resourceVersion: "13286" uid: 80d3e719-90f0-467c-93c8-8319f3a7ab77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9688cee1ef146 namespace: kserve-ci-e2e-test resourceVersion: "13289" uid: b9d816d7-dc6b-4f18-80c0-01e95fa0cefa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9688ceed3640d namespace: kserve-ci-e2e-test resourceVersion: "13292" uid: 350599d0-64bd-4038-b120-63cf5ae2f8e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:34:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:42Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9688ceedc9e89 namespace: kserve-ci-e2e-test resourceVersion: "13293" uid: d7521aae-de20-452a-8c4e-0fc208c6e8b9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.421s (2.421s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:44Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9688d7f38ca58 namespace: kserve-ci-e2e-test resourceVersion: "13319" uid: ddbce27a-73fd-49a3-be62-ef427e2950fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:44Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9688d829e662d namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: 1521d171-0e1b-48ae-a926-f499eb479b19 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:34:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:34:44Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9688d83121afd namespace: kserve-ci-e2e-test resourceVersion: "13322" uid: 4711133f-b0ba-4857-b1c5-87a49f0023f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:34:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:35:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:21Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9688dba2d0a16 namespace: kserve-ci-e2e-test resourceVersion: "13664" uid: 4e2f3daf-789c-4f99-bc12-4199eeea1f69 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9689de21f592e namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: eea87b23-1057-4260-b651-d192a1eb615f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9689de21fe574 namespace: kserve-ci-e2e-test resourceVersion: "14033" uid: 900c4289-4062-4db3-97f3-05b9b95029cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 6c2ac472-afc3-42bb-98cb-b32a47f4574c kind: Event lastTimestamp: "2026-04-24T21:35:56Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:56Z" name: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9.18a9689e4269dad0 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: 2cebd195-02b6-445f-b82c-b8a8693fd45c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9db4b-predictor-5f9575b58c namespace: kserve-ci-e2e-test resourceVersion: "12948" uid: 0b7e8f4f-caff-487f-b8c6-5d5c3d7006ca kind: Event lastTimestamp: "2026-04-24T21:34:24Z" message: 'Created pod: success-200-isvc-9db4b-predictor-5f9575b58c-pcsk9' metadata: creationTimestamp: "2026-04-24T21:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:24Z" name: success-200-isvc-9db4b-predictor-5f9575b58c.18a96888d659c137 namespace: kserve-ci-e2e-test resourceVersion: "12956" uid: e5fa42c3-b043-421f-829d-75b7621a73f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "12947" uid: 97533073-c197-4f2d-869d-cf70171fb1bd kind: Event lastTimestamp: "2026-04-24T21:34:24Z" message: Scaled up replica set success-200-isvc-9db4b-predictor-5f9575b58c from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:24Z" name: success-200-isvc-9db4b-predictor.18a96888d22b21b6 namespace: kserve-ci-e2e-test resourceVersion: "12950" uid: 74272bbb-9084-478c-8807-ef32299976f4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:34:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "12963" uid: 2749a962-478c-4ab7-809e-106e60ccafd9 kind: Event lastTimestamp: "2026-04-24T21:34:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:54Z" name: success-200-isvc-9db4b-predictor.18a9688c576a8d31 namespace: kserve-ci-e2e-test resourceVersion: "13439" uid: 34e41c79-e645-4ab4-a8cf-f77da9cf5a6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:34:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "12963" uid: 2749a962-478c-4ab7-809e-106e60ccafd9 kind: Event lastTimestamp: "2026-04-24T21:34:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:34:54Z" name: success-200-isvc-9db4b-predictor.18a9688c57d802c6 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: 986d42c0-28a2-4d08-8f7f-cddbe185c8dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:35:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13251" uid: 2749a962-478c-4ab7-809e-106e60ccafd9 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-9db4b-predictor.18a9689356db938d namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: d8b78a74-01ce-4617-a445-f4b9db266759 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:35:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9db4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13251" uid: 2749a962-478c-4ab7-809e-106e60ccafd9 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:55Z" name: success-200-isvc-9db4b-predictor.18a9689357511c50 namespace: kserve-ci-e2e-test resourceVersion: "13987" uid: c9a08666-b474-4558-bcf5-eaf77b24163f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:34:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9db4b namespace: kserve-ci-e2e-test resourceVersion: "12935" uid: 6c6c4fab-0cf1-4bcf-9141-39a5d77b6d55 kind: Event lastTimestamp: "2026-04-24T21:35:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:32Z" name: success-200-isvc-9db4b.18a96888d83df573 namespace: kserve-ci-e2e-test resourceVersion: "13752" uid: 7e0fdc5e-1355-4a21-bd4f-0605f8304e2d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9db4b namespace: kserve-ci-e2e-test resourceVersion: "12935" uid: 6c6c4fab-0cf1-4bcf-9141-39a5d77b6d55 kind: Event lastTimestamp: "2026-04-24T21:34:24Z" message: 'Failed to update status for InferenceService "success-200-isvc-9db4b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9db4b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:24Z" name: success-200-isvc-9db4b.18a96888d8b7d093 namespace: kserve-ci-e2e-test resourceVersion: "12968" uid: 6bf7cb57-b640-4069-8777-d2150fa81084 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:34:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9db4b namespace: kserve-ci-e2e-test resourceVersion: "12935" uid: 6c6c4fab-0cf1-4bcf-9141-39a5d77b6d55 kind: Event lastTimestamp: "2026-04-24T21:34:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9db4b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:34:24Z" name: success-200-isvc-9db4b.18a96888d8b81d66 namespace: kserve-ci-e2e-test resourceVersion: "12970" uid: 4ee8af5c-52c1-4614-9050-1e43d10191c3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9db4b namespace: kserve-ci-e2e-test resourceVersion: "13750" uid: 6c6c4fab-0cf1-4bcf-9141-39a5d77b6d55 kind: Event lastTimestamp: "2026-04-24T21:35:31Z" message: InferenceService [success-200-isvc-9db4b] is Ready metadata: creationTimestamp: "2026-04-24T21:35:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:31Z" name: success-200-isvc-9db4b.18a9689875729504 namespace: kserve-ci-e2e-test resourceVersion: "13751" uid: 64f6d67f-567a-4a07-b8c0-126714366650 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:46:20.587146Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19362" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:46:20Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a9692f78e6fa48 namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 7e8ae771-cd90-4b36-a902-cb16a15faea3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19386" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:46:21Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a9692f921ccd0b namespace: kserve-ci-e2e-test resourceVersion: "19420" uid: e23cd350-7c8a-447d-a488-262ca82d901e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a9692f93572b4f namespace: kserve-ci-e2e-test resourceVersion: "19423" uid: 209871ee-0aff-4971-899c-9d2bbf59c960 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a9692f96de4afb namespace: kserve-ci-e2e-test resourceVersion: "19430" uid: 45831e87-1e8c-484e-9216-3b398130e522 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a9692f975027b1 namespace: kserve-ci-e2e-test resourceVersion: "19431" uid: 9397f251-922a-4873-841b-587008a24ccb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a9692f975c1863 namespace: kserve-ci-e2e-test resourceVersion: "19432" uid: 577e0a14-836c-4c24-b02d-494182e23a61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a9692f9a2b9ab6 namespace: kserve-ci-e2e-test resourceVersion: "19433" uid: aa6f9df2-f49f-4bcb-a02a-7a24e5927cfa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:46:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:21Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a9692f9a9be08c namespace: kserve-ci-e2e-test resourceVersion: "19434" uid: 7e6272d4-0634-47d6-b649-eb0825d45f22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:46:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:46:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:59Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a9692fce356039 namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 7b627273-716b-49bb-8600-f76037441a63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:35Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a969b0ae94ae44 namespace: kserve-ci-e2e-test resourceVersion: "23737" uid: af081b6e-ca27-4d8e-83b0-f82f6ace5de9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:55:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: f409ce2b-d8c4-4c32-a764-32955142f14e kind: Event lastTimestamp: "2026-04-24T21:55:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:55:35Z" name: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t.18a969b0ae958d81 namespace: kserve-ci-e2e-test resourceVersion: "23739" uid: ca09a836-c02e-4fee-baf8-a492d202ebe9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a9fd5-predictor-864584f7cf namespace: kserve-ci-e2e-test resourceVersion: "19349" uid: 8f70af94-e0e5-4572-b4dc-4195a72ee484 kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: 'Created pod: success-200-isvc-a9fd5-predictor-864584f7cf-fgd5t' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:20Z" name: success-200-isvc-a9fd5-predictor-864584f7cf.18a9692f785f75fd namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: f0a51f17-db5a-432c-9e8f-41dcbb16800f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19344" uid: 9655cac3-5964-47e3-bed2-72271fac2a67 kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: Scaled up replica set success-200-isvc-a9fd5-predictor-864584f7cf from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:20Z" name: success-200-isvc-a9fd5-predictor.18a9692f765b101d namespace: kserve-ci-e2e-test resourceVersion: "19354" uid: e98c0025-3c9c-4699-b44a-3b1d095f8f6b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19347" uid: 268d5eac-3382-4b45-864d-5f655b64d842 kind: Event lastTimestamp: "2026-04-24T21:47:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:05Z" name: success-200-isvc-a9fd5-predictor.18a96932f4cee3eb namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: e8463ea2-8e5f-4d01-a8c5-b0319b866f67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:46:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19347" uid: 268d5eac-3382-4b45-864d-5f655b64d842 kind: Event lastTimestamp: "2026-04-24T21:47:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:05Z" name: success-200-isvc-a9fd5-predictor.18a96932f53eef25 namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: cda69b57-b944-47a1-8d54-b6a9d7807e42 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19585" uid: 268d5eac-3382-4b45-864d-5f655b64d842 kind: Event lastTimestamp: "2026-04-24T21:47:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:50Z" name: success-200-isvc-a9fd5-predictor.18a9693673f6ecc6 namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: f648627a-dc08-421c-a48c-3680b5ea22fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9fd5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19585" uid: 268d5eac-3382-4b45-864d-5f655b64d842 kind: Event lastTimestamp: "2026-04-24T21:47:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:50Z" name: success-200-isvc-a9fd5-predictor.18a96936746503b8 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: ef6fc745-fb40-4485-8cb1-ae95182ad3bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: 1ce26892-f993-4ebb-9d0d-0a91c5a8ad4e kind: Event lastTimestamp: "2026-04-24T21:47:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:09Z" name: success-200-isvc-a9fd5.18a9692f76242ca7 namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 60ab0e37-2aac-4d7f-a26c-fa995d8bc07b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: 1ce26892-f993-4ebb-9d0d-0a91c5a8ad4e kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: 'Failed to update status for InferenceService "success-200-isvc-a9fd5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a9fd5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:20Z" name: success-200-isvc-a9fd5.18a9692f76a4beda namespace: kserve-ci-e2e-test resourceVersion: "19356" uid: 1dfd117c-8be4-4776-a2e9-38d828cd55e3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: 1ce26892-f993-4ebb-9d0d-0a91c5a8ad4e kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a9fd5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:46:20Z" name: success-200-isvc-a9fd5.18a9692f76a51881 namespace: kserve-ci-e2e-test resourceVersion: "19358" uid: 0b91365b-bda7-4eee-a4df-2efe256a5408 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9fd5 namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: 1ce26892-f993-4ebb-9d0d-0a91c5a8ad4e kind: Event lastTimestamp: "2026-04-24T21:47:09Z" message: InferenceService [success-200-isvc-a9fd5] is Ready metadata: creationTimestamp: "2026-04-24T21:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:09Z" name: success-200-isvc-a9fd5.18a9693ac627d764 namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: b173bb1c-4ac4-4681-9f31-4d269b25ffe9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:47:00.435979Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19891" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a96938c0140ae4 namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 236e3704-bd5b-4345-a780-496dfc77950b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a96938d991c6e2 namespace: kserve-ci-e2e-test resourceVersion: "19955" uid: 3038c19c-05da-4533-bfba-8feb20766f1e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a96938dac3bd5f namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: fbdd26f5-c525-4e8d-a72c-a61d8a593f8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a96938ddc4b3b2 namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: bf768c46-0efb-4f0f-8e30-7bb0e6ccdbe6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a96938de39539c namespace: kserve-ci-e2e-test resourceVersion: "19961" uid: 57064673-730a-4866-ae22-a2a66cbfd073 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a96938de44005d namespace: kserve-ci-e2e-test resourceVersion: "19962" uid: 5e062123-222d-42e7-ac8f-aee2f62ca7ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a96938e0e94964 namespace: kserve-ci-e2e-test resourceVersion: "19963" uid: ac322ac7-fc99-4b94-957b-99de8313edc7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a96938e159cdfb namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: 02a421e0-9297-4610-be1b-37e443a44259 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:56:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:18Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a9693926dc739e namespace: kserve-ci-e2e-test resourceVersion: "24288" uid: fcdcb69f-a27b-4c0a-b985-d0e5bc00545b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a969b9f2330c31 namespace: kserve-ci-e2e-test resourceVersion: "24208" uid: 38db8c27-fd30-4a71-b4de-023f33214a01 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a969b9f23532e6 namespace: kserve-ci-e2e-test resourceVersion: "24210" uid: 6d748cc9-0792-4528-98d5-409963e7ecfa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9 namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 7eeb6208-ba7c-40e7-9ccf-0b38b2a3bc67 kind: Event lastTimestamp: "2026-04-24T21:56:18Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:18Z" name: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9.18a969ba9b892415 namespace: kserve-ci-e2e-test resourceVersion: "24286" uid: 91c63f9d-3e21-4240-8e65-3e8db7e84341 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d1134-predictor-b6cdf5487 namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 6efea724-9a28-41c3-a53c-ecdc5aaf6768 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: 'Created pod: success-200-isvc-d1134-predictor-b6cdf5487-xvsr9' metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor-b6cdf5487.18a96938bfae86d2 namespace: kserve-ci-e2e-test resourceVersion: "19894" uid: 5325285a-ccde-468a-854b-55564a08dbc9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "19869" uid: 58667620-0c7e-4072-bea3-69717079ef68 kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: Scaled up replica set success-200-isvc-d1134-predictor-b6cdf5487 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134-predictor.18a96938bd5d3f8b namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: 525e5abb-4c9e-42e2-adba-8a9d2ff81bb4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:47:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: a92f2c4d-c83a-4cd3-8ae3-f64655e484cc kind: Event lastTimestamp: "2026-04-24T21:47:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:30Z" name: success-200-isvc-d1134-predictor.18a9693c3c795b0a namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 463a13b4-622d-4862-9111-84b243f1215a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:47:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: a92f2c4d-c83a-4cd3-8ae3-f64655e484cc kind: Event lastTimestamp: "2026-04-24T21:47:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:30Z" name: success-200-isvc-d1134-predictor.18a9693c3cc7ea54 namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: 9e474415-02d0-4376-a7fa-a061681c5467 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:47:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: a92f2c4d-c83a-4cd3-8ae3-f64655e484cc kind: Event lastTimestamp: "2026-04-24T21:48:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:15Z" name: success-200-isvc-d1134-predictor.18a969433b8d519f namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 577d285d-ff1b-4eaf-b04a-4138ec21ca2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:47:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1134-predictor namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: a92f2c4d-c83a-4cd3-8ae3-f64655e484cc kind: Event lastTimestamp: "2026-04-24T21:48:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:15Z" name: success-200-isvc-d1134-predictor.18a969433c042fa0 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: d9f5c10d-a58f-4433-be63-10dc8b924df2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1134 namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 1fac379b-3ccf-4a21-90a1-9bd2fe7bd69d kind: Event lastTimestamp: "2026-04-24T21:47:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:48Z" name: success-200-isvc-d1134.18a96938bd321eba namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: 83e9e212-1ffe-4aec-b597-0c1c60ef4a5f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1134 namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 1fac379b-3ccf-4a21-90a1-9bd2fe7bd69d kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-d1134": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d1134": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134.18a96938bdab97b3 namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: 63fd1f40-f312-4ab2-8e83-9223cadb88f9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1134 namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 1fac379b-3ccf-4a21-90a1-9bd2fe7bd69d kind: Event lastTimestamp: "2026-04-24T21:47:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d1134": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:00Z" name: success-200-isvc-d1134.18a96938bdabe216 namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: bb105d8c-2b94-4f91-b99b-722df4351840 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1134 namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: 1fac379b-3ccf-4a21-90a1-9bd2fe7bd69d kind: Event lastTimestamp: "2026-04-24T21:47:48Z" message: InferenceService [success-200-isvc-d1134] is Ready metadata: creationTimestamp: "2026-04-24T21:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:48Z" name: success-200-isvc-d1134.18a96943e211f7ef namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 9a54a839-ea76-4305-9b98-5114510cdde3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:45:10.366389Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18464" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9691f1f6b3f29 namespace: kserve-ci-e2e-test resourceVersion: "18475" uid: 97caf338-9f55-4088-b921-1709fcaf098d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18492" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9691f3895f304 namespace: kserve-ci-e2e-test resourceVersion: "18523" uid: d877c29a-687c-40d8-b739-441bc0fe9ebd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9691f39daaa81 namespace: kserve-ci-e2e-test resourceVersion: "18525" uid: 289c8f57-5cb5-4912-9abb-1c8828fa53e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9691f3d6718ea namespace: kserve-ci-e2e-test resourceVersion: "18526" uid: 5af42dec-e9e5-41f7-a58d-1661fbcc2a3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9691f3df3ea2a namespace: kserve-ci-e2e-test resourceVersion: "18527" uid: 3a1612db-f7dd-4a28-a4fe-d2de736c7d24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9691f3e0018f8 namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: aea0a644-bd6d-47a0-b5cb-344e5bf2b44b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9691f40b77a80 namespace: kserve-ci-e2e-test resourceVersion: "18532" uid: df028b38-3844-43ef-9c2f-f3aeb560c27d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9691f41370b93 namespace: kserve-ci-e2e-test resourceVersion: "18533" uid: 53b05c4b-f774-4a05-b556-640bf742722e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:45:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:48Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9691fae69e106 namespace: kserve-ci-e2e-test resourceVersion: "18852" uid: 57e70f92-069b-430d-9dbc-730d97e02198 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:20Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9692f778eede6 namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 1028a960-0702-46a6-857f-c624c4cf8e65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:46:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:20Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9692f778f9f20 namespace: kserve-ci-e2e-test resourceVersion: "19364" uid: 075779e4-5c8f-40c0-9ae8-347386c9b9f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 214a2c49-4165-4a5f-a071-b7c230595d4c kind: Event lastTimestamp: "2026-04-24T21:46:23Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:46:23Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc.18a9693036a9ea09 namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: 11314239-7b1e-4928-a084-7bebda0d9203 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dd3fc-predictor-7dd9d668df namespace: kserve-ci-e2e-test resourceVersion: "18450" uid: 4581a995-7e02-405b-a580-b1b7442deb08 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: 'Created pod: success-200-isvc-dd3fc-predictor-7dd9d668df-b8wpc' metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor-7dd9d668df.18a9691f1ea2c818 namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: 5bf7adbc-d9f0-43be-afdf-4a79be9269a5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18442" uid: 503b316b-adbf-4118-8253-d2d4e8fb9465 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Scaled up replica set success-200-isvc-dd3fc-predictor-7dd9d668df from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc-predictor.18a9691f1aa7c901 namespace: kserve-ci-e2e-test resourceVersion: "18453" uid: 1ae5ceed-4137-4cdd-8675-d608ba3c2712 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18449" uid: 4ab51cae-3a27-4a54-8ce7-47b0b0d54e7b kind: Event lastTimestamp: "2026-04-24T21:45:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:40Z" name: success-200-isvc-dd3fc-predictor.18a9692299290b93 namespace: kserve-ci-e2e-test resourceVersion: "18784" uid: 975adec3-d7ff-4c3b-bf7c-927de5887c7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18449" uid: 4ab51cae-3a27-4a54-8ce7-47b0b0d54e7b kind: Event lastTimestamp: "2026-04-24T21:45:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:45:40Z" name: success-200-isvc-dd3fc-predictor.18a9692299933d98 namespace: kserve-ci-e2e-test resourceVersion: "18785" uid: 632f8613-fd15-4e29-950e-25308d50b899 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: 4ab51cae-3a27-4a54-8ce7-47b0b0d54e7b kind: Event lastTimestamp: "2026-04-24T21:46:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:10Z" name: success-200-isvc-dd3fc-predictor.18a969299755e293 namespace: kserve-ci-e2e-test resourceVersion: "19198" uid: ee648ae2-b875-49e1-aa3c-e9a5faa4c4ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:45:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd3fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: 4ab51cae-3a27-4a54-8ce7-47b0b0d54e7b kind: Event lastTimestamp: "2026-04-24T21:46:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:46:10Z" name: success-200-isvc-dd3fc-predictor.18a9692997c3ce71 namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: 0ee03413-adc2-4686-a80b-a3ee8654b0ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: 221b2c19-5e4e-413c-8e31-c871686010ec kind: Event lastTimestamp: "2026-04-24T21:45:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:58Z" name: success-200-isvc-dd3fc.18a9691f1a7435d1 namespace: kserve-ci-e2e-test resourceVersion: "19092" uid: 6866001b-d390-4576-8d5e-d9fc07330623 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: 221b2c19-5e4e-413c-8e31-c871686010ec kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-dd3fc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dd3fc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc.18a9691f1afcfd82 namespace: kserve-ci-e2e-test resourceVersion: "18456" uid: 57129989-acf9-4f2e-ba35-f5a0c6b5f21a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: 221b2c19-5e4e-413c-8e31-c871686010ec kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dd3fc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:10Z" name: success-200-isvc-dd3fc.18a9691f1afd71a4 namespace: kserve-ci-e2e-test resourceVersion: "18458" uid: a15bb7e5-a64d-4095-8915-fbff0da1ae6e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd3fc namespace: kserve-ci-e2e-test resourceVersion: "19087" uid: 221b2c19-5e4e-413c-8e31-c871686010ec kind: Event lastTimestamp: "2026-04-24T21:45:58Z" message: InferenceService [success-200-isvc-dd3fc] is Ready metadata: creationTimestamp: "2026-04-24T21:45:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:45:58Z" name: success-200-isvc-dd3fc.18a9692a69e349d0 namespace: kserve-ci-e2e-test resourceVersion: "19088" uid: 0bd24e91-4c03-4279-ac40-3533bd74c434 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:56:15.299065Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24190" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fc125-predictor-67497cd975-mrbkj to ip-10-0-132-124.ec2.internal metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a969b9f07ebbc2 namespace: kserve-ci-e2e-test resourceVersion: "24199" uid: 36176e96-5281-4813-8ee5-a7c7718c045d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24206" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a969ba09ae367c namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: c0989f11-3689-4704-9685-7dfbe8f63e09 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1450" already present on machine metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a969ba0b3f6461 namespace: kserve-ci-e2e-test resourceVersion: "24247" uid: 8deeb87c-337a-4d5a-af8e-e2889b0ab09b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a969ba0e9f4928 namespace: kserve-ci-e2e-test resourceVersion: "24248" uid: c2b52b52-f197-465d-930a-0648a09b5865 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a969ba0fb77767 namespace: kserve-ci-e2e-test resourceVersion: "24249" uid: 1db56f97-a766-4e55-bf7c-c63d9791fad6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a969ba0fc67b05 namespace: kserve-ci-e2e-test resourceVersion: "24250" uid: 1e8a6288-a80a-4023-866d-b1da02f3ca06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a969ba126e169d namespace: kserve-ci-e2e-test resourceVersion: "24254" uid: 6723654a-51eb-4be8-9b88-92e4966a1ff4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a969ba12e783db namespace: kserve-ci-e2e-test resourceVersion: "24255" uid: 99972fab-af2c-45c3-9b75-35faa5ba756f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T21:56:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:56:52Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a969ba4b30a9d9 namespace: kserve-ci-e2e-test resourceVersion: "24780" uid: 065b29cb-e00e-46ef-96ce-16cd2ee29b66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T22:13:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:13:34Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a96aabf7e9d85f namespace: kserve-ci-e2e-test resourceVersion: "32192" uid: e359f703-2778-4998-85c7-c13ad24ae273 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T22:13:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:13:34Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a96aabf7ea544c namespace: kserve-ci-e2e-test resourceVersion: "32193" uid: 4a58e48d-3e95-44c9-80b7-829b6d4d62aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:13:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc125-predictor-67497cd975-mrbkj namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 42e97302-b125-4622-918c-666a6a28d9c9 kind: Event lastTimestamp: "2026-04-24T22:13:37Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:13:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:13:37Z" name: success-200-isvc-fc125-predictor-67497cd975-mrbkj.18a96aacabf11a5e namespace: kserve-ci-e2e-test resourceVersion: "32258" uid: 6c791368-303d-4973-b5ca-be686e7cdaf4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-124.ec2.internal source: component: kubelet host: ip-10-0-132-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fc125-predictor-67497cd975 namespace: kserve-ci-e2e-test resourceVersion: "24177" uid: b11896ca-8542-433e-9449-cb55b6906585 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'Created pod: success-200-isvc-fc125-predictor-67497cd975-mrbkj' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor-67497cd975.18a969b9f0006030 namespace: kserve-ci-e2e-test resourceVersion: "24192" uid: 1143f7f7-a112-410e-aa0c-b04d74800ce6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24175" uid: e345fbc5-9061-46db-ac03-f3a119681947 kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: Scaled up replica set success-200-isvc-fc125-predictor-67497cd975 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125-predictor.18a969b9ed507040 namespace: kserve-ci-e2e-test resourceVersion: "24179" uid: 0039e592-bfe5-406d-96ac-7ec83da232a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:56:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: b3625dc3-238e-4729-a0e5-aa511338aefd kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:56:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:15Z" name: success-200-isvc-fc125-predictor.18a969bd6bea5b7f namespace: kserve-ci-e2e-test resourceVersion: "24993" uid: df040bfd-5df9-4076-91c3-7771424f20f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:56:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: b3625dc3-238e-4729-a0e5-aa511338aefd kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:56:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:15Z" name: success-200-isvc-fc125-predictor.18a969bd6c605b50 namespace: kserve-ci-e2e-test resourceVersion: "24995" uid: f13414bf-fe17-4bd8-9dab-8852446a75e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24411" uid: b3625dc3-238e-4729-a0e5-aa511338aefd kind: Event lastTimestamp: "2026-04-24T21:57:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:30Z" name: success-200-isvc-fc125-predictor.18a969cb6951001c namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: 2d20362c-01f5-4cb0-819b-46ccb090b336 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc125-predictor namespace: kserve-ci-e2e-test resourceVersion: "24411" uid: b3625dc3-238e-4729-a0e5-aa511338aefd kind: Event lastTimestamp: "2026-04-24T21:57:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:30Z" name: success-200-isvc-fc125-predictor.18a969cb69c4ebc5 namespace: kserve-ci-e2e-test resourceVersion: "25137" uid: 91e97755-2b61-4a32-8a9a-ceff4648813f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: 39ed1850-2d20-4d22-8c85-7620cdd41e8f kind: Event lastTimestamp: "2026-04-24T21:57:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:02Z" name: success-200-isvc-fc125.18a969b9ed1ebe9d namespace: kserve-ci-e2e-test resourceVersion: "24879" uid: 6e7ad5e0-9c48-42e3-af52-9ba68050c8b3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: 39ed1850-2d20-4d22-8c85-7620cdd41e8f kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-fc125": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fc125": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125.18a969b9ed713d4b namespace: kserve-ci-e2e-test resourceVersion: "24181" uid: 67b70190-0fb3-4851-abc3-deb0342f4d7b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:56:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: 39ed1850-2d20-4d22-8c85-7620cdd41e8f kind: Event lastTimestamp: "2026-04-24T21:56:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fc125": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:56:15Z" name: success-200-isvc-fc125.18a969b9ed71843c namespace: kserve-ci-e2e-test resourceVersion: "24182" uid: 6f809a17-511c-44f5-b20b-47dc51e0b6de reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24877" uid: 39ed1850-2d20-4d22-8c85-7620cdd41e8f kind: Event lastTimestamp: "2026-04-24T21:57:02Z" message: InferenceService [success-200-isvc-fc125] is Ready metadata: creationTimestamp: "2026-04-24T21:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:02Z" name: success-200-isvc-fc125.18a969c50723d540 namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 8947bf44-6da4-4363-ae3d-75445eddd329 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:36:55.398740Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6d3ba-548c7746dd-xgm2g namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: a9e392a8-a3cb-4106-91db-d76190d71448 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-6d3ba-548c7746dd-xgm2g to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:36:55Z" name: switch-graph-6d3ba-548c7746dd-xgm2g.18a968abe10c50cf namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: dcbf6c7b-901a-47f0-b389-30b6eb12ffd0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:55Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6d3ba-548c7746dd-xgm2g namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: a9e392a8-a3cb-4106-91db-d76190d71448 kind: Event lastTimestamp: "2026-04-24T21:36:55Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:36:55Z" name: switch-graph-6d3ba-548c7746dd-xgm2g.18a968abfa793d9e namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 8cadedcc-b94f-4edb-ab01-791620247742 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6d3ba} kind: Pod name: switch-graph-6d3ba-548c7746dd-xgm2g namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a9e392a8-a3cb-4106-91db-d76190d71448 kind: Event lastTimestamp: "2026-04-24T21:36:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:55Z" name: switch-graph-6d3ba-548c7746dd-xgm2g.18a968abfbcf3aee namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: b2493d92-3088-4929-9198-360ac15a1fb2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6d3ba} kind: Pod name: switch-graph-6d3ba-548c7746dd-xgm2g namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a9e392a8-a3cb-4106-91db-d76190d71448 kind: Event lastTimestamp: "2026-04-24T21:36:55Z" message: 'Created container: switch-graph-6d3ba' metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:55Z" name: switch-graph-6d3ba-548c7746dd-xgm2g.18a968abff2017cf namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 2f0aeced-6cd0-40b6-a92c-c1eb20bc2c27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6d3ba} kind: Pod name: switch-graph-6d3ba-548c7746dd-xgm2g namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a9e392a8-a3cb-4106-91db-d76190d71448 kind: Event lastTimestamp: "2026-04-24T21:36:55Z" message: Started container switch-graph-6d3ba metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:55Z" name: switch-graph-6d3ba-548c7746dd-xgm2g.18a968abff94da50 namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: db485b8e-4dca-475f-8333-4c68c1f5df6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6d3ba} kind: Pod name: switch-graph-6d3ba-548c7746dd-xgm2g namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a9e392a8-a3cb-4106-91db-d76190d71448 kind: Event lastTimestamp: "2026-04-24T21:45:10Z" message: Stopping container switch-graph-6d3ba metadata: creationTimestamp: "2026-04-24T21:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:10Z" name: switch-graph-6d3ba-548c7746dd-xgm2g.18a9691f1475db37 namespace: kserve-ci-e2e-test resourceVersion: "18426" uid: 2faa73f1-77f4-48b0-be0e-669ef23d2a9e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6d3ba} kind: Pod name: switch-graph-6d3ba-548c7746dd-xgm2g namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a9e392a8-a3cb-4106-91db-d76190d71448 kind: Event lastTimestamp: "2026-04-24T21:45:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:45:37Z" name: switch-graph-6d3ba-548c7746dd-xgm2g.18a9691f96e1e15c namespace: kserve-ci-e2e-test resourceVersion: "18760" uid: a11ae53a-7c43-4226-af0b-3fca3748b407 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-6d3ba-548c7746dd namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: 8de5edb6-96f7-45f4-8b92-014d4f12399b kind: Event lastTimestamp: "2026-04-24T21:36:55Z" message: 'Created pod: switch-graph-6d3ba-548c7746dd-xgm2g' metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:55Z" name: switch-graph-6d3ba-548c7746dd.18a968abe063bf00 namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 8b842eaf-d784-4cf2-a135-34ebf206ccf3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: 14f6a89a-2abb-4941-9cf6-38bcd4f78544 kind: Event lastTimestamp: "2026-04-24T21:36:55Z" message: Scaled up replica set switch-graph-6d3ba-548c7746dd from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:36:55Z" name: switch-graph-6d3ba.18a968abdf986432 namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: 3578ad28-1181-4abd-8b29-7f132cd17cf2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:36:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: 2811f4d2-24ef-441a-ade8-c0dc145544e0 kind: Event lastTimestamp: "2026-04-24T21:36:55Z" message: InferenceGraph [switch-graph-6d3ba] is Ready metadata: creationTimestamp: "2026-04-24T21:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:36:55Z" name: switch-graph-6d3ba.18a968abe3e68421 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: b1ec5a38-69e3-4f38-8887-0e7857111150 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:37:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: e07f40c3-8c9d-4de0-bfc2-7357037e3b28 kind: Event lastTimestamp: "2026-04-24T21:37:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:25Z" name: switch-graph-6d3ba.18a968af5e2a19c7 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 1e68e6e5-c469-40ff-9374-36ad212a5b8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:37:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6d3ba namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: e07f40c3-8c9d-4de0-bfc2-7357037e3b28 kind: Event lastTimestamp: "2026-04-24T21:37:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:25Z" name: switch-graph-6d3ba.18a968af5e99a237 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 09b55f7e-f809-45b3-b391-5fc427724a84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:35:45.060013Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9db4b-8484996d65-m4klk namespace: kserve-ci-e2e-test resourceVersion: "13884" uid: b4e5e7bc-e9dd-48c1-b2b3-05cfb35fbee3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9db4b-8484996d65-m4klk to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:35:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:35:45Z" name: switch-graph-9db4b-8484996d65-m4klk.18a9689b80888f01 namespace: kserve-ci-e2e-test resourceVersion: "13890" uid: 4ba280d5-9d3d-4b13-823e-767ecd0f15cc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:45Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9db4b-8484996d65-m4klk namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: b4e5e7bc-e9dd-48c1-b2b3-05cfb35fbee3 kind: Event lastTimestamp: "2026-04-24T21:35:45Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:35:45Z" name: switch-graph-9db4b-8484996d65-m4klk.18a9689b9a53f08e namespace: kserve-ci-e2e-test resourceVersion: "13905" uid: 18301b7e-0f7b-48ac-ab37-0e7d085c805c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9db4b} kind: Pod name: switch-graph-9db4b-8484996d65-m4klk namespace: kserve-ci-e2e-test resourceVersion: "13886" uid: b4e5e7bc-e9dd-48c1-b2b3-05cfb35fbee3 kind: Event lastTimestamp: "2026-04-24T21:35:45Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" metadata: creationTimestamp: "2026-04-24T21:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:45Z" name: switch-graph-9db4b-8484996d65-m4klk.18a9689b9c8468ab namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 7ef0976f-a98b-4051-9b95-693239aaa085 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9db4b} kind: Pod name: switch-graph-9db4b-8484996d65-m4klk namespace: kserve-ci-e2e-test resourceVersion: "13886" uid: b4e5e7bc-e9dd-48c1-b2b3-05cfb35fbee3 kind: Event lastTimestamp: "2026-04-24T21:35:47Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" in 1.98s (1.98s including waiting). Image size: 216346202 bytes.' metadata: creationTimestamp: "2026-04-24T21:35:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:47Z" name: switch-graph-9db4b-8484996d65-m4klk.18a9689c128b35d7 namespace: kserve-ci-e2e-test resourceVersion: "13922" uid: 21950f30-de9d-4f66-8db7-0832a08da67b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9db4b} kind: Pod name: switch-graph-9db4b-8484996d65-m4klk namespace: kserve-ci-e2e-test resourceVersion: "13886" uid: b4e5e7bc-e9dd-48c1-b2b3-05cfb35fbee3 kind: Event lastTimestamp: "2026-04-24T21:35:47Z" message: 'Created container: switch-graph-9db4b' metadata: creationTimestamp: "2026-04-24T21:35:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:47Z" name: switch-graph-9db4b-8484996d65-m4klk.18a9689c1612bc18 namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: e03873f2-3e9f-47fb-9e34-180c0989279e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9db4b} kind: Pod name: switch-graph-9db4b-8484996d65-m4klk namespace: kserve-ci-e2e-test resourceVersion: "13886" uid: b4e5e7bc-e9dd-48c1-b2b3-05cfb35fbee3 kind: Event lastTimestamp: "2026-04-24T21:35:47Z" message: Started container switch-graph-9db4b metadata: creationTimestamp: "2026-04-24T21:35:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:47Z" name: switch-graph-9db4b-8484996d65-m4klk.18a9689c16758d07 namespace: kserve-ci-e2e-test resourceVersion: "13924" uid: fe5f23e6-3c82-4f0f-bd54-fb03e6c3fd9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9db4b} kind: Pod name: switch-graph-9db4b-8484996d65-m4klk namespace: kserve-ci-e2e-test resourceVersion: "13886" uid: b4e5e7bc-e9dd-48c1-b2b3-05cfb35fbee3 kind: Event lastTimestamp: "2026-04-24T21:35:55Z" message: Stopping container switch-graph-9db4b metadata: creationTimestamp: "2026-04-24T21:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:35:55Z" name: switch-graph-9db4b-8484996d65-m4klk.18a9689dd9b2e694 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: d383f9b7-c1e6-4c98-a111-58d6efe1fe33 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9db4b} kind: Pod name: switch-graph-9db4b-8484996d65-m4klk namespace: kserve-ci-e2e-test resourceVersion: "13886" uid: b4e5e7bc-e9dd-48c1-b2b3-05cfb35fbee3 kind: Event lastTimestamp: "2026-04-24T21:36:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:36:24Z" name: switch-graph-9db4b-8484996d65-m4klk.18a9689ec8755de0 namespace: kserve-ci-e2e-test resourceVersion: "14387" uid: 3c4561df-33a0-421e-8a20-7afe593f6ce6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9db4b-8484996d65 namespace: kserve-ci-e2e-test resourceVersion: "13880" uid: 227c1de4-d51e-4999-92f1-7a1f975902fd kind: Event lastTimestamp: "2026-04-24T21:35:45Z" message: 'Created pod: switch-graph-9db4b-8484996d65-m4klk' metadata: creationTimestamp: "2026-04-24T21:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:45Z" name: switch-graph-9db4b-8484996d65.18a9689b7fee2c8d namespace: kserve-ci-e2e-test resourceVersion: "13885" uid: 089d2a42-fc3b-438a-a8d3-4486502b19c0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9db4b namespace: kserve-ci-e2e-test resourceVersion: "13879" uid: e74ff9f5-a210-4717-b1eb-f5fa5b24f540 kind: Event lastTimestamp: "2026-04-24T21:35:45Z" message: Scaled up replica set switch-graph-9db4b-8484996d65 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:35:45Z" name: switch-graph-9db4b.18a9689b7ec5166b namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: 5a8ddf35-eaa8-4f47-a839-11734117b817 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:35:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9db4b namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: a61ebbe4-8dac-401c-a310-446e0f9dd2c5 kind: Event lastTimestamp: "2026-04-24T21:35:45Z" message: InferenceGraph [switch-graph-9db4b] is Ready metadata: creationTimestamp: "2026-04-24T21:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:35:45Z" name: switch-graph-9db4b.18a9689b857d8b37 namespace: kserve-ci-e2e-test resourceVersion: "13901" uid: 575c19a1-2ac2-4cef-8792-0576ca1a2cc7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:57:15.315419Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-fc125-5c9566c648-xg996 namespace: kserve-ci-e2e-test resourceVersion: "24986" uid: 97a1ecf0-a7f7-49a7-8370-f7975294b638 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-fc125-5c9566c648-xg996 to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:57:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:57:15Z" name: switch-graph-fc125-5c9566c648-xg996.18a969c7e9bfa39f namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: c379f9c0-4dad-4798-87f6-3b15572e9519 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b879b5f7b-5cpl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:15Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-fc125-5c9566c648-xg996 namespace: kserve-ci-e2e-test resourceVersion: "24996" uid: 97a1ecf0-a7f7-49a7-8370-f7975294b638 kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:57:15Z" name: switch-graph-fc125-5c9566c648-xg996.18a969c803a19722 namespace: kserve-ci-e2e-test resourceVersion: "25010" uid: bb720a25-448b-4010-93b4-4c9f642df75c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc125} kind: Pod name: switch-graph-fc125-5c9566c648-xg996 namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 97a1ecf0-a7f7-49a7-8370-f7975294b638 kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bfb5b2782772ad46be0a5c41e64aded1dd42f09d6e2e6f1f1c3194223df058cb" already present on machine metadata: creationTimestamp: "2026-04-24T21:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:15Z" name: switch-graph-fc125-5c9566c648-xg996.18a969c8057d6c56 namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: ba042d46-3577-4b3b-a22c-32be0eedb3b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc125} kind: Pod name: switch-graph-fc125-5c9566c648-xg996 namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 97a1ecf0-a7f7-49a7-8370-f7975294b638 kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: 'Created container: switch-graph-fc125' metadata: creationTimestamp: "2026-04-24T21:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:15Z" name: switch-graph-fc125-5c9566c648-xg996.18a969c808d23f98 namespace: kserve-ci-e2e-test resourceVersion: "25013" uid: 04521d3f-97f5-49ed-8c36-778871a02a16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc125} kind: Pod name: switch-graph-fc125-5c9566c648-xg996 namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 97a1ecf0-a7f7-49a7-8370-f7975294b638 kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: Started container switch-graph-fc125 metadata: creationTimestamp: "2026-04-24T21:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:57:15Z" name: switch-graph-fc125-5c9566c648-xg996.18a969c8094fdf1d namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 2f40acef-264a-42a0-a21a-37c1923a6e3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc125} kind: Pod name: switch-graph-fc125-5c9566c648-xg996 namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 97a1ecf0-a7f7-49a7-8370-f7975294b638 kind: Event lastTimestamp: "2026-04-24T22:13:34Z" message: Stopping container switch-graph-fc125 metadata: creationTimestamp: "2026-04-24T22:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:13:34Z" name: switch-graph-fc125-5c9566c648-xg996.18a96aabf0717027 namespace: kserve-ci-e2e-test resourceVersion: "32177" uid: 13ff6095-eb47-4314-94b7-4980c19f7ce3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:13:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc125} kind: Pod name: switch-graph-fc125-5c9566c648-xg996 namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 97a1ecf0-a7f7-49a7-8370-f7975294b638 kind: Event lastTimestamp: "2026-04-24T22:13:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:13:56Z" name: switch-graph-fc125-5c9566c648-xg996.18a96aac7782e24d namespace: kserve-ci-e2e-test resourceVersion: "32409" uid: abe4d03d-098f-4752-83b1-48fa0791f3bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-fc125-5c9566c648 namespace: kserve-ci-e2e-test resourceVersion: "24982" uid: 6d821e92-0b23-418c-84c4-3c76abcc0c40 kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: 'Created pod: switch-graph-fc125-5c9566c648-xg996' metadata: creationTimestamp: "2026-04-24T21:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:15Z" name: switch-graph-fc125-5c9566c648.18a969c7e90a1719 namespace: kserve-ci-e2e-test resourceVersion: "24990" uid: 8f841511-cccb-492b-a919-6d21aea65fb5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24980" uid: 1c9d331c-1ed7-4ec6-8e5f-bebabcf2b8a6 kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: Scaled up replica set switch-graph-fc125-5c9566c648 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:15Z" name: switch-graph-fc125.18a969c7e847e73a namespace: kserve-ci-e2e-test resourceVersion: "24983" uid: e41eb88f-cd99-4459-adb3-326f8a28f772 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: f9e586f9-109f-42e5-b257-0d62ba7a930a kind: Event lastTimestamp: "2026-04-24T21:57:15Z" message: InferenceGraph [switch-graph-fc125] is Ready metadata: creationTimestamp: "2026-04-24T21:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:57:15Z" name: switch-graph-fc125.18a969c7eea24d51 namespace: kserve-ci-e2e-test resourceVersion: "25000" uid: 8e34bb55-eb3a-468d-8cb9-983a5f5d3ce8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 50eb7445-b6f9-4d64-9827-63d924ed8d5d kind: Event lastTimestamp: "2026-04-24T21:57:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:30Z" name: switch-graph-fc125.18a969cb67083107 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 0a67a657-ab42-4c6b-ac9e-b85c7b6df9d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-fc125 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 50eb7445-b6f9-4d64-9827-63d924ed8d5d kind: Event lastTimestamp: "2026-04-24T21:57:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:30Z" name: switch-graph-fc125.18a969cb67809f31 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 9bb7f68d-bc24-445d-b476-1cfd7d59a8ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-fc125 namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: 50eb7445-b6f9-4d64-9827-63d924ed8d5d kind: Event lastTimestamp: "2026-04-24T21:57:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:45Z" name: switch-graph-fc125.18a969cee6e6aefa namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 21031b1e-f858-48c0-a4a6-dc3a7c2aef9d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-fc125 namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: 50eb7445-b6f9-4d64-9827-63d924ed8d5d kind: Event lastTimestamp: "2026-04-24T21:57:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:57:45Z" name: switch-graph-fc125.18a969cee7673a8d namespace: kserve-ci-e2e-test resourceVersion: "25265" uid: ef774d97-4114-4edf-aa58-266f896f35a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32421"