--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:07:49.599257Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-19f83-b5859dffb-4pxf7 namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: d6a6472d-1553-4fe1-8f9c-13d2dee6d93a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-19f83-b5859dffb-4pxf7 to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:07:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:07:49Z" name: ensemble-graph-19f83-b5859dffb-4pxf7.189dfe83a3fcbd50 namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: 5f2f69e3-6764-4fd2-b154-cbc3b178b081 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:50Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-19f83-b5859dffb-4pxf7 namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: d6a6472d-1553-4fe1-8f9c-13d2dee6d93a kind: Event lastTimestamp: "2026-03-18T17:07:50Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:07:50Z" name: ensemble-graph-19f83-b5859dffb-4pxf7.189dfe83bd1f3072 namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: 87fd3078-200f-4c16-aa34-aa962466faf5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-19f83} kind: Pod name: ensemble-graph-19f83-b5859dffb-4pxf7 namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: d6a6472d-1553-4fe1-8f9c-13d2dee6d93a kind: Event lastTimestamp: "2026-03-18T17:07:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T17:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:50Z" name: ensemble-graph-19f83-b5859dffb-4pxf7.189dfe83bebfd93f namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: ec02bbd9-55e9-431b-b1c0-5ff8fa12f6e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-19f83} kind: Pod name: ensemble-graph-19f83-b5859dffb-4pxf7 namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: d6a6472d-1553-4fe1-8f9c-13d2dee6d93a kind: Event lastTimestamp: "2026-03-18T17:07:50Z" message: 'Created container: ensemble-graph-19f83' metadata: creationTimestamp: "2026-03-18T17:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:50Z" name: ensemble-graph-19f83-b5859dffb-4pxf7.189dfe83c18f1da0 namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: 2f0a0737-5175-43b7-b8ca-627d52b5aec3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-19f83} kind: Pod name: ensemble-graph-19f83-b5859dffb-4pxf7 namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: d6a6472d-1553-4fe1-8f9c-13d2dee6d93a kind: Event lastTimestamp: "2026-03-18T17:07:50Z" message: Started container ensemble-graph-19f83 metadata: creationTimestamp: "2026-03-18T17:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:50Z" name: ensemble-graph-19f83-b5859dffb-4pxf7.189dfe83c2039231 namespace: kserve-ci-e2e-test resourceVersion: "20257" uid: d40207e3-8b5e-411a-a6e4-4e50c978800f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-19f83} kind: Pod name: ensemble-graph-19f83-b5859dffb-4pxf7 namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: d6a6472d-1553-4fe1-8f9c-13d2dee6d93a kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: Stopping container ensemble-graph-19f83 metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:04Z" name: ensemble-graph-19f83-b5859dffb-4pxf7.189dfef6cdd6367b namespace: kserve-ci-e2e-test resourceVersion: "22511" uid: 45d85bc1-3dfc-49dd-9138-0f6539314d66 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-19f83} kind: Pod name: ensemble-graph-19f83-b5859dffb-4pxf7 namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: d6a6472d-1553-4fe1-8f9c-13d2dee6d93a kind: Event lastTimestamp: "2026-03-18T17:16:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T17:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:31Z" name: ensemble-graph-19f83-b5859dffb-4pxf7.189dfef740b7d750 namespace: kserve-ci-e2e-test resourceVersion: "22757" uid: 20c48928-a4c8-4de5-b316-c052ca122809 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-19f83-b5859dffb namespace: kserve-ci-e2e-test resourceVersion: "20230" uid: d8f979b6-3313-431b-bf64-25226b39cfdd kind: Event lastTimestamp: "2026-03-18T17:07:49Z" message: 'Created pod: ensemble-graph-19f83-b5859dffb-4pxf7' metadata: creationTimestamp: "2026-03-18T17:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:49Z" name: ensemble-graph-19f83-b5859dffb.189dfe83a3a5bd07 namespace: kserve-ci-e2e-test resourceVersion: "20235" uid: d39d1dfb-d02d-4692-a0b5-af62fd581b1a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-19f83 namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 3f8e9241-24cc-437b-94f9-819ad9accb7b kind: Event lastTimestamp: "2026-03-18T17:07:49Z" message: Scaled up replica set ensemble-graph-19f83-b5859dffb from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:49Z" name: ensemble-graph-19f83.189dfe83a2c5241b namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: e1c92930-decb-432b-9e47-203b55bc82f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-19f83 namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: 5f5a24cb-321c-416e-a9fa-774bf2c2dd28 kind: Event lastTimestamp: "2026-03-18T17:07:49Z" message: InferenceGraph [ensemble-graph-19f83] is Ready metadata: creationTimestamp: "2026-03-18T17:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:07:49Z" name: ensemble-graph-19f83.189dfe83a6025c62 namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: 96374daf-325b-49ca-94a9-485712edcf77 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-19f83 namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: 0a5d5c50-dfb0-4d8f-b0e3-83d18fa8e4ea kind: Event lastTimestamp: "2026-03-18T17:08:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:04Z" name: ensemble-graph-19f83.189dfe87226e1a1c namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 6a061c1c-dd51-4a97-8ee7-852d371fec6b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-19f83 namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: 0a5d5c50-dfb0-4d8f-b0e3-83d18fa8e4ea kind: Event lastTimestamp: "2026-03-18T17:08:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:04Z" name: ensemble-graph-19f83.189dfe87234ef84b namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 11598b91-3fd1-4837-8588-5ad8150b5988 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-19f83 namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: 0a5d5c50-dfb0-4d8f-b0e3-83d18fa8e4ea kind: Event lastTimestamp: "2026-03-18T17:08:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:19Z" name: ensemble-graph-19f83.189dfe8aa1f610fb namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: a84ef9be-67a2-4c36-a99a-530e58e3b8a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-19f83 namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: 0a5d5c50-dfb0-4d8f-b0e3-83d18fa8e4ea kind: Event lastTimestamp: "2026-03-18T17:08:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:19Z" name: ensemble-graph-19f83.189dfe8aa25cc18b namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 26e7e9b7-cd2c-4689-8dd0-8055b9e008eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:06:19.203758Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4ddda-559cdcff4b-9mqdv namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 6974693a-2c12-408d-b531-df87aeae8976 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-4ddda-559cdcff4b-9mqdv to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:06:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:06:19Z" name: ensemble-graph-4ddda-559cdcff4b-9mqdv.189dfe6e97fee21d namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 0a8c3a70-b7f7-4094-a6fb-7f01c337b19e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:19Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4ddda-559cdcff4b-9mqdv namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 6974693a-2c12-408d-b531-df87aeae8976 kind: Event lastTimestamp: "2026-03-18T17:06:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-4ddda-serving-cert" not found' metadata: creationTimestamp: "2026-03-18T17:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:19Z" name: ensemble-graph-4ddda-559cdcff4b-9mqdv.189dfe6ea7bc79f4 namespace: kserve-ci-e2e-test resourceVersion: "19380" uid: 7b2ebafd-36c2-47f2-acdb-be93dc1dd3a1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:20Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4ddda-559cdcff4b-9mqdv namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 6974693a-2c12-408d-b531-df87aeae8976 kind: Event lastTimestamp: "2026-03-18T17:06:20Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:06:20Z" name: ensemble-graph-4ddda-559cdcff4b-9mqdv.189dfe6ed4e57b35 namespace: kserve-ci-e2e-test resourceVersion: "19385" uid: ac4b90c4-1472-4b0d-a78f-ea1542485490 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4ddda} kind: Pod name: ensemble-graph-4ddda-559cdcff4b-9mqdv namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 6974693a-2c12-408d-b531-df87aeae8976 kind: Event lastTimestamp: "2026-03-18T17:06:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T17:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:20Z" name: ensemble-graph-4ddda-559cdcff4b-9mqdv.189dfe6ed63c60f9 namespace: kserve-ci-e2e-test resourceVersion: "19387" uid: ae247b4d-9d54-41da-9c57-222cbde034b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4ddda} kind: Pod name: ensemble-graph-4ddda-559cdcff4b-9mqdv namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 6974693a-2c12-408d-b531-df87aeae8976 kind: Event lastTimestamp: "2026-03-18T17:06:20Z" message: 'Created container: ensemble-graph-4ddda' metadata: creationTimestamp: "2026-03-18T17:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:20Z" name: ensemble-graph-4ddda-559cdcff4b-9mqdv.189dfe6ed93c6009 namespace: kserve-ci-e2e-test resourceVersion: "19388" uid: 377744d7-9f4e-4346-9f25-fc9aa3c84660 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4ddda} kind: Pod name: ensemble-graph-4ddda-559cdcff4b-9mqdv namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 6974693a-2c12-408d-b531-df87aeae8976 kind: Event lastTimestamp: "2026-03-18T17:06:20Z" message: Started container ensemble-graph-4ddda metadata: creationTimestamp: "2026-03-18T17:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:20Z" name: ensemble-graph-4ddda-559cdcff4b-9mqdv.189dfe6ed9ae8719 namespace: kserve-ci-e2e-test resourceVersion: "19389" uid: f0257d30-fbc7-4da2-8410-8634ff45cb99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4ddda} kind: Pod name: ensemble-graph-4ddda-559cdcff4b-9mqdv namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 6974693a-2c12-408d-b531-df87aeae8976 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: Stopping container ensemble-graph-4ddda metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:29Z" name: ensemble-graph-4ddda-559cdcff4b-9mqdv.189dfe70eef7612f namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 49336dcd-2903-4750-873b-ec68fe980fe5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4ddda} kind: Pod name: ensemble-graph-4ddda-559cdcff4b-9mqdv namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 6974693a-2c12-408d-b531-df87aeae8976 kind: Event lastTimestamp: "2026-03-18T17:06:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T17:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:56Z" name: ensemble-graph-4ddda-559cdcff4b-9mqdv.189dfe718903b113 namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: e880c550-f3d3-4af7-b03c-9e44abf6c086 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-4ddda-559cdcff4b namespace: kserve-ci-e2e-test resourceVersion: "19357" uid: 1332cd59-7ef1-457f-b6b9-da4da006f168 kind: Event lastTimestamp: "2026-03-18T17:06:19Z" message: 'Created pod: ensemble-graph-4ddda-559cdcff4b-9mqdv' metadata: creationTimestamp: "2026-03-18T17:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:19Z" name: ensemble-graph-4ddda-559cdcff4b.189dfe6e979a32f9 namespace: kserve-ci-e2e-test resourceVersion: "19362" uid: 992d0872-b15c-4c6b-9c7a-2fc12344d64a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-4ddda namespace: kserve-ci-e2e-test resourceVersion: "19355" uid: ea04b193-3432-4bf5-8776-938fd7198797 kind: Event lastTimestamp: "2026-03-18T17:06:19Z" message: Scaled up replica set ensemble-graph-4ddda-559cdcff4b from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:19Z" name: ensemble-graph-4ddda.189dfe6e96b0106b namespace: kserve-ci-e2e-test resourceVersion: "19358" uid: ded8818e-fa35-4c33-8f81-f8380b486739 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4ddda namespace: kserve-ci-e2e-test resourceVersion: "19371" uid: 8a7abd6d-5b4a-48f4-ac0c-342a2366a99e kind: Event lastTimestamp: "2026-03-18T17:06:19Z" message: InferenceGraph [ensemble-graph-4ddda] is Ready metadata: creationTimestamp: "2026-03-18T17:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:19Z" name: ensemble-graph-4ddda.189dfe6e9b8c9210 namespace: kserve-ci-e2e-test resourceVersion: "19373" uid: 8889f63f-58a9-42be-a3b2-ffe74fedc36c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4ddda namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: 8a7abd6d-5b4a-48f4-ac0c-342a2366a99e kind: Event lastTimestamp: "2026-03-18T17:06:19Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-4ddda": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-4ddda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:19Z" name: ensemble-graph-4ddda.189dfe6e9daa2c94 namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: e29d3753-9303-4af0-90fd-623144a855ca reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4ddda namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: 8a7abd6d-5b4a-48f4-ac0c-342a2366a99e kind: Event lastTimestamp: "2026-03-18T17:06:19Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-4ddda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:19Z" name: ensemble-graph-4ddda.189dfe6e9daa6970 namespace: kserve-ci-e2e-test resourceVersion: "19376" uid: a160e3cb-69cf-4d0c-9732-a295abb75104 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:16:04.853065Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw namespace: kserve-ci-e2e-test resourceVersion: "22582" uid: bcc10674-da0b-4889-8b6f-5f40931c86e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:16:04Z" name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw.189dfef6f36a27f1 namespace: kserve-ci-e2e-test resourceVersion: "22588" uid: 6bc7c6bb-581f-4120-9c08-6ca926822f19 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw namespace: kserve-ci-e2e-test resourceVersion: "22593" uid: bcc10674-da0b-4889-8b6f-5f40931c86e3 kind: Event lastTimestamp: "2026-03-18T17:16:05Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:16:05Z" name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw.189dfef70d0f0071 namespace: kserve-ci-e2e-test resourceVersion: "22607" uid: c8869997-578e-443b-bfba-35e57b59b77b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw namespace: kserve-ci-e2e-test resourceVersion: "22586" uid: bcc10674-da0b-4889-8b6f-5f40931c86e3 kind: Event lastTimestamp: "2026-03-18T17:16:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:05Z" name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw.189dfef70e42e54f namespace: kserve-ci-e2e-test resourceVersion: "22611" uid: d2907999-9972-4205-86ab-2d2614c24003 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw namespace: kserve-ci-e2e-test resourceVersion: "22586" uid: bcc10674-da0b-4889-8b6f-5f40931c86e3 kind: Event lastTimestamp: "2026-03-18T17:16:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:05Z" name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw.189dfef71450cac9 namespace: kserve-ci-e2e-test resourceVersion: "22615" uid: f262c669-caa9-4708-af53-ec6443680ce5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw namespace: kserve-ci-e2e-test resourceVersion: "22586" uid: bcc10674-da0b-4889-8b6f-5f40931c86e3 kind: Event lastTimestamp: "2026-03-18T17:16:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:05Z" name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw.189dfef714c16794 namespace: kserve-ci-e2e-test resourceVersion: "22616" uid: b41a612d-feaf-44d3-b08f-0cfac7d8731d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw namespace: kserve-ci-e2e-test resourceVersion: "22586" uid: bcc10674-da0b-4889-8b6f-5f40931c86e3 kind: Event lastTimestamp: "2026-03-18T17:17:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:36Z" name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw.189dfef72ef34480 namespace: kserve-ci-e2e-test resourceVersion: "23712" uid: 9603ec5b-1061-47f3-bdc1-0f9e0f45d191 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw namespace: kserve-ci-e2e-test resourceVersion: "22586" uid: bcc10674-da0b-4889-8b6f-5f40931c86e3 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw.189dff0bfa2c5253 namespace: kserve-ci-e2e-test resourceVersion: "23648" uid: 94bbfd61-371c-4686-897d-2be564067952 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0c4f8-predictor-5fc4bc99fd namespace: kserve-ci-e2e-test resourceVersion: "22570" uid: 68816f81-f5f4-498e-aa58-a69834c0b5bb kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: 'Created pod: error-404-isvc-0c4f8-predictor-5fc4bc99fd-spxdw' metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:04Z" name: error-404-isvc-0c4f8-predictor-5fc4bc99fd.189dfef6f2de82e4 namespace: kserve-ci-e2e-test resourceVersion: "22585" uid: db3ecfda-4595-4402-be48-ba2f9d096969 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: df7acc1a-c0c1-4ae9-adfb-9cec0dd23127 kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: Scaled up replica set error-404-isvc-0c4f8-predictor-5fc4bc99fd from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:04Z" name: error-404-isvc-0c4f8-predictor.189dfef6ef07616e namespace: kserve-ci-e2e-test resourceVersion: "22575" uid: 74c15b0f-1086-40a3-9baf-b8e29bfe6ca8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22579" uid: d20e51bf-10c3-4223-85d7-9a335e6710a9 kind: Event lastTimestamp: "2026-03-18T17:16:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:16:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:19Z" name: error-404-isvc-0c4f8-predictor.189dfefa709f92a9 namespace: kserve-ci-e2e-test resourceVersion: "22706" uid: f3199d7a-3a75-4d59-979b-66c3c9bd7ba0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22579" uid: d20e51bf-10c3-4223-85d7-9a335e6710a9 kind: Event lastTimestamp: "2026-03-18T17:16:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:16:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:19Z" name: error-404-isvc-0c4f8-predictor.189dfefa70fad80e namespace: kserve-ci-e2e-test resourceVersion: "22708" uid: 7f03512b-944d-456a-9842-ef8c049e285e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:16:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22707" uid: d20e51bf-10c3-4223-85d7-9a335e6710a9 kind: Event lastTimestamp: "2026-03-18T17:17:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:34Z" name: error-404-isvc-0c4f8-predictor.189dfefdef920cc3 namespace: kserve-ci-e2e-test resourceVersion: "23593" uid: 0747df45-bf2e-4f54-80e1-f8894a43e856 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:16:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22707" uid: d20e51bf-10c3-4223-85d7-9a335e6710a9 kind: Event lastTimestamp: "2026-03-18T17:17:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:34Z" name: error-404-isvc-0c4f8-predictor.189dfefdeff047a9 namespace: kserve-ci-e2e-test resourceVersion: "23594" uid: 5623e30e-1be0-4ab0-9b9f-cc597c6fb115 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "22546" uid: 6e6a7715-5d6a-4bf5-8017-b9d7ab676d57 kind: Event lastTimestamp: "2026-03-18T17:17:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:06Z" name: error-404-isvc-0c4f8.189dfef6f214786e namespace: kserve-ci-e2e-test resourceVersion: "23102" uid: 9a751aee-08da-456f-9351-6a738bb32ddf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "22546" uid: 6e6a7715-5d6a-4bf5-8017-b9d7ab676d57 kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-0c4f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0c4f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:16:04Z" name: error-404-isvc-0c4f8.189dfef6f25aa1ac namespace: kserve-ci-e2e-test resourceVersion: "22581" uid: 8f3eecae-92b0-4bdc-8386-3b68506860a9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "22546" uid: 6e6a7715-5d6a-4bf5-8017-b9d7ab676d57 kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0c4f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:16:04Z" name: error-404-isvc-0c4f8.189dfef6f25ae42c namespace: kserve-ci-e2e-test resourceVersion: "22583" uid: cb945111-1c91-4f18-b96f-28cb68574613 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "23100" uid: 6e6a7715-5d6a-4bf5-8017-b9d7ab676d57 kind: Event lastTimestamp: "2026-03-18T17:17:06Z" message: InferenceService [error-404-isvc-0c4f8] is Ready metadata: creationTimestamp: "2026-03-18T17:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:06Z" name: error-404-isvc-0c4f8.189dff05667df062 namespace: kserve-ci-e2e-test resourceVersion: "23101" uid: 6241addf-431e-47d6-9453-ef748ead06dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:05:39.149723Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-18dc8-predictor-646c887d76-n8862 namespace: kserve-ci-e2e-test resourceVersion: "19075" uid: a5935591-7466-4118-ad77-a4f0fb921c33 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-18dc8-predictor-646c887d76-n8862 to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:05:39Z" name: error-404-isvc-18dc8-predictor-646c887d76-n8862.189dfe654496d092 namespace: kserve-ci-e2e-test resourceVersion: "19079" uid: e22f126c-1b7c-44e1-9362-b240042639c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-18dc8-predictor-646c887d76-n8862 namespace: kserve-ci-e2e-test resourceVersion: "19085" uid: a5935591-7466-4118-ad77-a4f0fb921c33 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:05:39Z" name: error-404-isvc-18dc8-predictor-646c887d76-n8862.189dfe655db978d4 namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: 5ded347f-8a75-4021-a54e-aa5ca779bc40 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-18dc8-predictor-646c887d76-n8862 namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: a5935591-7466-4118-ad77-a4f0fb921c33 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:39Z" name: error-404-isvc-18dc8-predictor-646c887d76-n8862.189dfe655f09d1d9 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: f964ff4b-b80d-4af7-affd-144b1f3bab4a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-18dc8-predictor-646c887d76-n8862 namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: a5935591-7466-4118-ad77-a4f0fb921c33 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:39Z" name: error-404-isvc-18dc8-predictor-646c887d76-n8862.189dfe65658b524e namespace: kserve-ci-e2e-test resourceVersion: "19107" uid: 35b0c098-73cb-4c30-bba3-c580bb368110 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-18dc8-predictor-646c887d76-n8862 namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: a5935591-7466-4118-ad77-a4f0fb921c33 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:39Z" name: error-404-isvc-18dc8-predictor-646c887d76-n8862.189dfe6565fbaa51 namespace: kserve-ci-e2e-test resourceVersion: "19110" uid: 8b2b6123-182f-4d60-af01-74642c77db75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-18dc8-predictor-646c887d76-n8862 namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: a5935591-7466-4118-ad77-a4f0fb921c33 kind: Event lastTimestamp: "2026-03-18T17:07:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:10Z" name: error-404-isvc-18dc8-predictor-646c887d76-n8862.189dfe65761ac5f7 namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: c6af3e1a-b57b-4e26-b5f9-f04377ef2e50 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-18dc8-predictor-646c887d76-n8862 namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: a5935591-7466-4118-ad77-a4f0fb921c33 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:09Z" name: error-404-isvc-18dc8-predictor-646c887d76-n8862.189dfe7a4c750a64 namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 03c2845e-6ab5-42c5-9ad4-91a00668b9f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-18dc8-predictor-646c887d76 namespace: kserve-ci-e2e-test resourceVersion: "19056" uid: cb9b31ce-4426-4ada-ae14-4f3b12fc4ed3 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: 'Created pod: error-404-isvc-18dc8-predictor-646c887d76-n8862' metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:05:39Z" name: error-404-isvc-18dc8-predictor-646c887d76.189dfe654430ce1d namespace: kserve-ci-e2e-test resourceVersion: "19076" uid: eed0c423-b852-4ee7-ae8b-85ea09cc3cc6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-18dc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19054" uid: 94229e5c-6cbe-4e6d-b5d1-7bed088e8b80 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: Scaled up replica set error-404-isvc-18dc8-predictor-646c887d76 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:05:39Z" name: error-404-isvc-18dc8-predictor.189dfe65406e3190 namespace: kserve-ci-e2e-test resourceVersion: "19060" uid: d2155e4e-39b7-40f3-bfa3-dbbde8fa129f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:05:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-18dc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19055" uid: d7597516-b2d0-415a-aa0a-eef943cddfda kind: Event lastTimestamp: "2026-03-18T17:06:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:05:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:09Z" name: error-404-isvc-18dc8-predictor.189dfe68bf069bb1 namespace: kserve-ci-e2e-test resourceVersion: "19295" uid: e821b436-e2e6-4d4a-bfde-441f8f3cb635 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:05:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-18dc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19055" uid: d7597516-b2d0-415a-aa0a-eef943cddfda kind: Event lastTimestamp: "2026-03-18T17:06:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:05:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:09Z" name: error-404-isvc-18dc8-predictor.189dfe68bf6436b3 namespace: kserve-ci-e2e-test resourceVersion: "19297" uid: 9a81353c-2afd-47c6-bf4a-c85f0de4213e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-18dc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: d7597516-b2d0-415a-aa0a-eef943cddfda kind: Event lastTimestamp: "2026-03-18T17:06:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:54Z" name: error-404-isvc-18dc8-predictor.189dfe6fbcadabeb namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: e5781660-735a-47ae-978e-ccf9837df743 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-18dc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: d7597516-b2d0-415a-aa0a-eef943cddfda kind: Event lastTimestamp: "2026-03-18T17:06:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:54Z" name: error-404-isvc-18dc8-predictor.189dfe6fbd10778d namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: 81c59d16-b8b0-471d-b46b-c423deb86166 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-18dc8 namespace: kserve-ci-e2e-test resourceVersion: "19046" uid: 0b2aa75d-4b12-4e96-b028-6ff9962169a9 kind: Event lastTimestamp: "2026-03-18T17:06:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:41Z" name: error-404-isvc-18dc8.189dfe6540694fcf namespace: kserve-ci-e2e-test resourceVersion: "19652" uid: cf57ee83-082c-4dd8-9f8f-e2cc9920f6b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-18dc8 namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: 0b2aa75d-4b12-4e96-b028-6ff9962169a9 kind: Event lastTimestamp: "2026-03-18T17:06:41Z" message: InferenceService [error-404-isvc-18dc8] is Ready metadata: creationTimestamp: "2026-03-18T17:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:41Z" name: error-404-isvc-18dc8.189dfe73adbc052d namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: 43750fbb-2c40-4024-9af3-aab3655e3f1d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:06:29.749960Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: f34256db-4192-491f-918c-db97261a9d3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-19f83-predictor-6d757459d6-rjs8t to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:06:29Z" name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t.189dfe710c9922e4 namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: 7378d34f-d6b6-4d43-a108-768b3cd14908 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: f34256db-4192-491f-918c-db97261a9d3b kind: Event lastTimestamp: "2026-03-18T17:06:30Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:06:30Z" name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t.189dfe7125a2af45 namespace: kserve-ci-e2e-test resourceVersion: "19560" uid: 0eb1be5c-4879-44ca-b526-59f913c238a7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: f34256db-4192-491f-918c-db97261a9d3b kind: Event lastTimestamp: "2026-03-18T17:06:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:30Z" name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t.189dfe7126c7943c namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: 0f8870bc-e082-4ec3-8455-e687f0e15077 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: f34256db-4192-491f-918c-db97261a9d3b kind: Event lastTimestamp: "2026-03-18T17:06:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:30Z" name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t.189dfe712d7262f4 namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: bb5bb3e0-fc91-4270-9684-a0aa956aae81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: f34256db-4192-491f-918c-db97261a9d3b kind: Event lastTimestamp: "2026-03-18T17:06:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:30Z" name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t.189dfe712de7eac6 namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: e73ab5e3-4571-4593-9903-84b468665e00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T17:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: f34256db-4192-491f-918c-db97261a9d3b kind: Event lastTimestamp: "2026-03-18T17:07:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:22Z" name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t.189dfe715f52f226 namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: b02a4724-e3fe-4426-a85e-66ba78540569 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: f34256db-4192-491f-918c-db97261a9d3b kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:04Z" name: error-404-isvc-19f83-predictor-6d757459d6-rjs8t.189dfef6eee88963 namespace: kserve-ci-e2e-test resourceVersion: "22574" uid: 9b064191-c63d-4a05-b60b-4ab5c7990bc6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-19f83-predictor-6d757459d6 namespace: kserve-ci-e2e-test resourceVersion: "19526" uid: a2927fff-77bc-44bf-938d-da5d6e2b1560 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'Created pod: error-404-isvc-19f83-predictor-6d757459d6-rjs8t' metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:29Z" name: error-404-isvc-19f83-predictor-6d757459d6.189dfe710c362f92 namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: fafc3db1-03fe-46b5-9743-6594dd5242b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19523" uid: 0f6ce468-2b86-40e0-9e6b-c6d79eff469a kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: Scaled up replica set error-404-isvc-19f83-predictor-6d757459d6 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:29Z" name: error-404-isvc-19f83-predictor.189dfe7108dbe9d8 namespace: kserve-ci-e2e-test resourceVersion: "19527" uid: 0b3e14d5-f067-4121-bd31-e4f2aa286fc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:06:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19525" uid: a92c8e17-36ad-4aaf-903b-a1af088bfcbc kind: Event lastTimestamp: "2026-03-18T17:06:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:59Z" name: error-404-isvc-19f83-predictor.189dfe74874643a9 namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 01a1bb5e-7c11-40e4-a2bc-32da96960690 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:06:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19525" uid: a92c8e17-36ad-4aaf-903b-a1af088bfcbc kind: Event lastTimestamp: "2026-03-18T17:06:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:59Z" name: error-404-isvc-19f83-predictor.189dfe7487a2dce0 namespace: kserve-ci-e2e-test resourceVersion: "19791" uid: b8bfabfd-116c-4eca-8f7d-6ece9f2e4475 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:07:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19676" uid: a92c8e17-36ad-4aaf-903b-a1af088bfcbc kind: Event lastTimestamp: "2026-03-18T17:07:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:59Z" name: error-404-isvc-19f83-predictor.189dfe7b84f93a3a namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: 420c75c1-348f-4f17-817f-5fbb088dfeb8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:07:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19676" uid: a92c8e17-36ad-4aaf-903b-a1af088bfcbc kind: Event lastTimestamp: "2026-03-18T17:07:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:59Z" name: error-404-isvc-19f83-predictor.189dfe7b85721973 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: e494aef1-da0f-4f01-b678-6fbc5ad1f714 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-19f83 namespace: kserve-ci-e2e-test resourceVersion: "19505" uid: 226122a8-80c5-4f63-ac3c-771863d8c146 kind: Event lastTimestamp: "2026-03-18T17:07:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:07:32Z" name: error-404-isvc-19f83.189dfe7108bd6c96 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 2509e6c0-81f4-48cd-8690-926bf4b7e4b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-19f83 namespace: kserve-ci-e2e-test resourceVersion: "19505" uid: 226122a8-80c5-4f63-ac3c-771863d8c146 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-19f83": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-19f83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:29Z" name: error-404-isvc-19f83.189dfe710b89d35f namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: a79e713f-bcd8-46bf-aeb7-373aaae8c651 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-19f83 namespace: kserve-ci-e2e-test resourceVersion: "19505" uid: 226122a8-80c5-4f63-ac3c-771863d8c146 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-19f83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:29Z" name: error-404-isvc-19f83.189dfe710b8a2129 namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: 906dc3c4-ff1c-4ed4-9174-e5125b227ecc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-19f83 namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: 226122a8-80c5-4f63-ac3c-771863d8c146 kind: Event lastTimestamp: "2026-03-18T17:07:32Z" message: InferenceService [error-404-isvc-19f83] is Ready metadata: creationTimestamp: "2026-03-18T17:07:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:07:32Z" name: error-404-isvc-19f83.189dfe7f9a1fc905 namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: de836914-859a-4d50-915a-80c06894845a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:04:59.445121Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8 namespace: kserve-ci-e2e-test resourceVersion: "18739" uid: 57cebae5-c6d8-48fe-896f-6bf4e093c294 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8 to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:04:59Z" name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8.189dfe5c0602a60f namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 2ebfb91b-e0c5-4eea-a358-219daa7c07c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8 namespace: kserve-ci-e2e-test resourceVersion: "18757" uid: 57cebae5-c6d8-48fe-896f-6bf4e093c294 kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:04:59Z" name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8.189dfe5c1f4ee481 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: 4a0901db-4f92-46bf-81a8-bfce267880a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8 namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: 57cebae5-c6d8-48fe-896f-6bf4e093c294 kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:04:59Z" name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8.189dfe5c207b36e5 namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 8d2d24c3-e429-4cc9-b50d-7ee1ad6f6f36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8 namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: 57cebae5-c6d8-48fe-896f-6bf4e093c294 kind: Event lastTimestamp: "2026-03-18T17:05:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:00Z" name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8.189dfe5c27821f98 namespace: kserve-ci-e2e-test resourceVersion: "18779" uid: 78f4164a-c4fb-41ea-943b-ffd3b684b3f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8 namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: 57cebae5-c6d8-48fe-896f-6bf4e093c294 kind: Event lastTimestamp: "2026-03-18T17:05:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:00Z" name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8.189dfe5c27fd4cd9 namespace: kserve-ci-e2e-test resourceVersion: "18780" uid: 74507a43-06ac-4e3a-9702-9d9510163650 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8 namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: 57cebae5-c6d8-48fe-896f-6bf4e093c294 kind: Event lastTimestamp: "2026-03-18T17:06:31Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:31Z" name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8.189dfe5c5a436090 namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: b0dbfe64-78a8-4a46-a41e-e4fd8421291c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8 namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: 57cebae5-c6d8-48fe-896f-6bf4e093c294 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:29Z" name: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8.189dfe70fd40aa0a namespace: kserve-ci-e2e-test resourceVersion: "19485" uid: 5a6599c4-b2d2-4f57-a981-66a65f668dd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4ddda-predictor-6f79885fcd namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: 434c1dc6-25ff-4a51-8154-012b224c22a3 kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: 'Created pod: error-404-isvc-4ddda-predictor-6f79885fcd-tpnm8' metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:04:59Z" name: error-404-isvc-4ddda-predictor-6f79885fcd.189dfe5c05a02f3b namespace: kserve-ci-e2e-test resourceVersion: "18741" uid: 3277e4eb-dce1-4644-80c8-ef79a6a25f56 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18730" uid: 5c0b2025-45fe-499e-8092-be7d01b66f64 kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: Scaled up replica set error-404-isvc-4ddda-predictor-6f79885fcd from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:04:59Z" name: error-404-isvc-4ddda-predictor.189dfe5c0228f8cd namespace: kserve-ci-e2e-test resourceVersion: "18735" uid: 09966a68-bf7f-4035-9376-102ff6e2bd49 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18731" uid: cbea1d3e-a344-48be-8f95-bdd308dd848a kind: Event lastTimestamp: "2026-03-18T17:05:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:05:14Z" name: error-404-isvc-4ddda-predictor.189dfe5f809ec3f8 namespace: kserve-ci-e2e-test resourceVersion: "18866" uid: 838e6481-766a-4b3a-ab26-0c466d2e58a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18731" uid: cbea1d3e-a344-48be-8f95-bdd308dd848a kind: Event lastTimestamp: "2026-03-18T17:05:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:05:14Z" name: error-404-isvc-4ddda-predictor.189dfe5f810f805c namespace: kserve-ci-e2e-test resourceVersion: "18868" uid: 4f237eff-63e4-448a-9a9f-9f88ba03d8f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18867" uid: cbea1d3e-a344-48be-8f95-bdd308dd848a kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:29Z" name: error-404-isvc-4ddda-predictor.189dfe62ffaf7ad8 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: 487766bd-9d82-43cc-b54e-783d394c833e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18867" uid: cbea1d3e-a344-48be-8f95-bdd308dd848a kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:29Z" name: error-404-isvc-4ddda-predictor.189dfe63001282f4 namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 3fba4314-ede0-43f6-baee-b92319a39534 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4ddda namespace: kserve-ci-e2e-test resourceVersion: "18724" uid: 70cfe3f6-edc5-40ab-8f84-4aa734727145 kind: Event lastTimestamp: "2026-03-18T17:06:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:01Z" name: error-404-isvc-4ddda.189dfe5c01fa0324 namespace: kserve-ci-e2e-test resourceVersion: "19258" uid: 877d5db0-9855-449d-8e02-9ea5718f30c6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4ddda namespace: kserve-ci-e2e-test resourceVersion: "19256" uid: 70cfe3f6-edc5-40ab-8f84-4aa734727145 kind: Event lastTimestamp: "2026-03-18T17:06:01Z" message: InferenceService [error-404-isvc-4ddda] is Ready metadata: creationTimestamp: "2026-03-18T17:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:01Z" name: error-404-isvc-4ddda.189dfe6a92072aef namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: c1328328-417b-49f3-8e6a-eab2ae5546bc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:17:35.344592Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs namespace: kserve-ci-e2e-test resourceVersion: "23664" uid: 10c64850-8784-44ca-a6cd-26d488fa32fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs.189dff0c0521404a namespace: kserve-ci-e2e-test resourceVersion: "23670" uid: 4910c8e7-caf9-4bb6-838a-136151c88a0d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs namespace: kserve-ci-e2e-test resourceVersion: "23674" uid: 10c64850-8784-44ca-a6cd-26d488fa32fe kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs.189dff0c1e560ad3 namespace: kserve-ci-e2e-test resourceVersion: "23691" uid: 6554f0fe-3ae4-469b-8e18-e1842455dee7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs namespace: kserve-ci-e2e-test resourceVersion: "23666" uid: 10c64850-8784-44ca-a6cd-26d488fa32fe kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs.189dff0c1fb430c6 namespace: kserve-ci-e2e-test resourceVersion: "23693" uid: 5e041eed-2685-4cd8-8508-e3dfb51bd040 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs namespace: kserve-ci-e2e-test resourceVersion: "23666" uid: 10c64850-8784-44ca-a6cd-26d488fa32fe kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs.189dff0c267e03c3 namespace: kserve-ci-e2e-test resourceVersion: "23696" uid: bb675023-6ecd-4722-8cd9-e011858f4b94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs namespace: kserve-ci-e2e-test resourceVersion: "23666" uid: 10c64850-8784-44ca-a6cd-26d488fa32fe kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs.189dff0c26f86757 namespace: kserve-ci-e2e-test resourceVersion: "23697" uid: 935601c3-4d94-4532-9c62-a0d2befcfd70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T17:17:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs namespace: kserve-ci-e2e-test resourceVersion: "23666" uid: 10c64850-8784-44ca-a6cd-26d488fa32fe kind: Event lastTimestamp: "2026-03-18T17:18:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:27Z" name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs.189dff0c343d4589 namespace: kserve-ci-e2e-test resourceVersion: "24047" uid: d7a6b86f-0924-4750-b95c-d36ea90e2c9a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs namespace: kserve-ci-e2e-test resourceVersion: "23666" uid: 10c64850-8784-44ca-a6cd-26d488fa32fe kind: Event lastTimestamp: "2026-03-18T17:27:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:27:10Z" name: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs.189dff91d66fcfa8 namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: d218efe6-98a8-492d-92cd-cf47c7e1cd32 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6eae8-predictor-77dd5796b6 namespace: kserve-ci-e2e-test resourceVersion: "23654" uid: d48d6801-c7a4-485c-a30b-1ac9d7b1e0ba kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: 'Created pod: error-404-isvc-6eae8-predictor-77dd5796b6-zz7vs' metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-6eae8-predictor-77dd5796b6.189dff0c04c62c50 namespace: kserve-ci-e2e-test resourceVersion: "23665" uid: 3847cc07-4b7a-493b-9e0d-e619eb606f09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23653" uid: 0baaa674-a078-48e4-9def-68b92ced9d86 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: Scaled up replica set error-404-isvc-6eae8-predictor-77dd5796b6 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-6eae8-predictor.189dff0bfb8f9f1e namespace: kserve-ci-e2e-test resourceVersion: "23656" uid: 90678f73-ccfd-4cf2-8185-1670ea5c6389 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:17:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23655" uid: a07dd033-bd29-4b94-84d4-38846e2b4d95 kind: Event lastTimestamp: "2026-03-18T17:18:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:20Z" name: error-404-isvc-6eae8-predictor.189dff0f7a382bc3 namespace: kserve-ci-e2e-test resourceVersion: "24016" uid: 9a4e3064-cd04-4a83-9930-a8891ae70d9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:17:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23655" uid: a07dd033-bd29-4b94-84d4-38846e2b4d95 kind: Event lastTimestamp: "2026-03-18T17:18:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:20Z" name: error-404-isvc-6eae8-predictor.189dff0f7a772bc4 namespace: kserve-ci-e2e-test resourceVersion: "24017" uid: d685fdb8-a65f-447f-bc9d-3ce8477ba925 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:18:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23809" uid: a07dd033-bd29-4b94-84d4-38846e2b4d95 kind: Event lastTimestamp: "2026-03-18T17:19:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:19:05Z" name: error-404-isvc-6eae8-predictor.189dff19f6a02fa4 namespace: kserve-ci-e2e-test resourceVersion: "24288" uid: c16ae523-8567-4624-97b2-57eee59fbb0d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:18:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23809" uid: a07dd033-bd29-4b94-84d4-38846e2b4d95 kind: Event lastTimestamp: "2026-03-18T17:19:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:19:05Z" name: error-404-isvc-6eae8-predictor.189dff19f70089a7 namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 6c846ec7-3b3c-4ff3-9da9-96149d558bc7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "23635" uid: 4df35add-04af-4141-a68b-bcaf16cab7c6 kind: Event lastTimestamp: "2026-03-18T17:18:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:18:37Z" name: error-404-isvc-6eae8.189dff0bfba73281 namespace: kserve-ci-e2e-test resourceVersion: "24117" uid: cd7956a7-dbce-4f22-a847-072307b21c11 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "23635" uid: 4df35add-04af-4141-a68b-bcaf16cab7c6 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-6eae8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6eae8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-6eae8.189dff0bfc1c8de3 namespace: kserve-ci-e2e-test resourceVersion: "23659" uid: 4ee99dbb-57e4-4231-b84e-44c7c8736462 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "23635" uid: 4df35add-04af-4141-a68b-bcaf16cab7c6 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6eae8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:35Z" name: error-404-isvc-6eae8.189dff0bfc1cbf38 namespace: kserve-ci-e2e-test resourceVersion: "23661" uid: 3ca129ea-fdd9-41fe-b92e-e43f399cd310 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 4df35add-04af-4141-a68b-bcaf16cab7c6 kind: Event lastTimestamp: "2026-03-18T17:18:37Z" message: InferenceService [error-404-isvc-6eae8] is Ready metadata: creationTimestamp: "2026-03-18T17:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:18:37Z" name: error-404-isvc-6eae8.189dff1a6eeb85a7 namespace: kserve-ci-e2e-test resourceVersion: "24116" uid: 62aca9cb-d7be-461a-a64f-0091018c0b43 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:07:09.455792Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5 namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: 0af702de-e71b-4aec-9584-4868398f7bc0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5 to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:07:09Z" name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5.189dfe7a4b400dee namespace: kserve-ci-e2e-test resourceVersion: "19934" uid: 2c473c59-8e61-46b5-ba71-cd73121c7c84 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5 namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 0af702de-e71b-4aec-9584-4868398f7bc0 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:07:09Z" name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5.189dfe7a647f4bdf namespace: kserve-ci-e2e-test resourceVersion: "19962" uid: b1b2d371-3597-4e67-b9a2-cf21470c582e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5 namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: 0af702de-e71b-4aec-9584-4868398f7bc0 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:09Z" name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5.189dfe7a65b39baa namespace: kserve-ci-e2e-test resourceVersion: "19965" uid: 3ed6a194-1b9b-4240-992c-bff7f9c9fd95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5 namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: 0af702de-e71b-4aec-9584-4868398f7bc0 kind: Event lastTimestamp: "2026-03-18T17:07:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:10Z" name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5.189dfe7a6bf7ca2e namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 8968d930-bb99-4cec-8a11-ce5a94a60e06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5 namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: 0af702de-e71b-4aec-9584-4868398f7bc0 kind: Event lastTimestamp: "2026-03-18T17:07:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:10Z" name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5.189dfe7a6c64ce19 namespace: kserve-ci-e2e-test resourceVersion: "19971" uid: 2096f489-e488-4008-bcad-2d61b3caad86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T17:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5 namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: 0af702de-e71b-4aec-9584-4868398f7bc0 kind: Event lastTimestamp: "2026-03-18T17:16:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:51Z" name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5.189dfe7a7b544f56 namespace: kserve-ci-e2e-test resourceVersion: "22994" uid: 97878fee-c538-4a01-b3bd-c36cb2882679 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5 namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: 0af702de-e71b-4aec-9584-4868398f7bc0 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5.189dff001ad4d125 namespace: kserve-ci-e2e-test resourceVersion: "22878" uid: 366c63ea-31cb-447a-a160-c1e3d54c0889 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8d83a-predictor-6d6f8878fb namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: b0713d21-2a36-4b49-9a93-7259a0463df7 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: 'Created pod: error-404-isvc-8d83a-predictor-6d6f8878fb-tbwp5' metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:09Z" name: error-404-isvc-8d83a-predictor-6d6f8878fb.189dfe7a4aac0aee namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: d34481be-6927-44de-8996-9aa596e1939f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8d83a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: 6ec94c56-c55c-484f-842e-722fff18a79d kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: Scaled up replica set error-404-isvc-8d83a-predictor-6d6f8878fb from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:09Z" name: error-404-isvc-8d83a-predictor.189dfe7a46a59ded namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: 3fed5d9d-6e4f-4698-bcc8-a7d69d2497da reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d83a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 6702c68b-ef3a-4e78-94fc-0b80019f366e kind: Event lastTimestamp: "2026-03-18T17:08:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:24Z" name: error-404-isvc-8d83a-predictor.189dfe7dc561fa8e namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: d4ae7214-a94f-4180-9b75-07635ef8201a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d83a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 6702c68b-ef3a-4e78-94fc-0b80019f366e kind: Event lastTimestamp: "2026-03-18T17:08:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:24Z" name: error-404-isvc-8d83a-predictor.189dfe7dc5a48cb4 namespace: kserve-ci-e2e-test resourceVersion: "20463" uid: df84f9e9-f80c-40fd-8e46-16a1b001b87b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d83a namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: ab33f622-fcee-4a87-b856-36efbe3cf62f kind: Event lastTimestamp: "2026-03-18T17:08:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:08:11Z" name: error-404-isvc-8d83a.189dfe7a46be4be1 namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 24f87c4a-66d4-4fc2-a4ab-96e114c42211 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d83a namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: ab33f622-fcee-4a87-b856-36efbe3cf62f kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-8d83a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d83a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:07:09Z" name: error-404-isvc-8d83a.189dfe7a497a7158 namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 55133a14-d128-4152-90e5-4ae6786ed634 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d83a namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: ab33f622-fcee-4a87-b856-36efbe3cf62f kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d83a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:07:09Z" name: error-404-isvc-8d83a.189dfe7a497ac03b namespace: kserve-ci-e2e-test resourceVersion: "19924" uid: dc31289a-990b-4adb-980a-2f6cf4ca42ae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d83a namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: ab33f622-fcee-4a87-b856-36efbe3cf62f kind: Event lastTimestamp: "2026-03-18T17:08:11Z" message: InferenceService [error-404-isvc-8d83a] is Ready metadata: creationTimestamp: "2026-03-18T17:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:08:11Z" name: error-404-isvc-8d83a.189dfe88b2fc6a29 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 8856bec4-dd7e-45f8-947f-0ea32f6ec767 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:56:04.367952Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 42e0ab27-c561-4c38-b5be-e004773cde91 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:56:04Z" name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp.189dfddf70ed0699 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: 583792a9-ddd1-43d9-ae55-f32b7fa62baa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 42e0ab27-c561-4c38-b5be-e004773cde91 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:56:04Z" name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp.189dfddf89ebd502 namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 1686a95d-42ea-4ac0-92cd-528cdd5d2bc6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 42e0ab27-c561-4c38-b5be-e004773cde91 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:04Z" name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp.189dfddf8b15c830 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 8c3834bd-459a-4847-aad3-9945f363f388 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 42e0ab27-c561-4c38-b5be-e004773cde91 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:04Z" name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp.189dfddf90ecf84b namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: e143998d-372a-4601-b51c-84fca95db0d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 42e0ab27-c561-4c38-b5be-e004773cde91 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:04Z" name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp.189dfddf91580d68 namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: f47aa43b-761a-423f-bc51-96fdf9d794c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T16:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 42e0ab27-c561-4c38-b5be-e004773cde91 kind: Event lastTimestamp: "2026-03-18T16:56:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T16:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:56Z" name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp.189dfddfa6a5d95d namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: 4be55478-e55e-4bd9-8bf9-66d3e8043601 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 42e0ab27-c561-4c38-b5be-e004773cde91 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:39Z" name: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp.189dfe6542baeb11 namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 5ebcf9b1-4778-459b-ae15-632f5f191a55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9913d-predictor-5ccb69bd64 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: de84e5c5-1826-4e05-a89a-bd5da82f489f kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: 'Created pod: error-404-isvc-9913d-predictor-5ccb69bd64-h6pgp' metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:04Z" name: error-404-isvc-9913d-predictor-5ccb69bd64.189dfddf708a9210 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: ba152308-b0d6-4f58-8ecb-25f1c40ccf5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: f6c36523-7b0a-45fc-aeff-4f251dfbef8f kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Scaled up replica set error-404-isvc-9913d-predictor-5ccb69bd64 from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:04Z" name: error-404-isvc-9913d-predictor.189dfddf6f418e78 namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 984fbc56-75cd-4a72-a9c9-e22135d2723e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: e2783c72-97a5-4902-ac40-4327b4f052ba kind: Event lastTimestamp: "2026-03-18T16:56:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:34Z" name: error-404-isvc-9913d-predictor.189dfde2edc7b45b namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 3b372dbe-4138-4366-9ec5-ea605d25c2b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: e2783c72-97a5-4902-ac40-4327b4f052ba kind: Event lastTimestamp: "2026-03-18T16:56:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:34Z" name: error-404-isvc-9913d-predictor.189dfde2ee28aa44 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: f29d3963-8b49-414d-999c-b8352eab42c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T16:56:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: e2783c72-97a5-4902-ac40-4327b4f052ba kind: Event lastTimestamp: "2026-03-18T16:57:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:34Z" name: error-404-isvc-9913d-predictor.189dfde9eb7e8aaa namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 4934bc89-1c41-4237-ac98-07e6c5f79737 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T16:56:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: e2783c72-97a5-4902-ac40-4327b4f052ba kind: Event lastTimestamp: "2026-03-18T16:57:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:34Z" name: error-404-isvc-9913d-predictor.189dfde9ebde123b namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: 730d43fd-4204-460f-9a30-0c2c3302a2bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9913d namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: 55ada20d-8e14-4ef1-985a-5d0f54b55f24 kind: Event lastTimestamp: "2026-03-18T16:57:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:57:06Z" name: error-404-isvc-9913d.189dfddf6f1f459c namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 1d81232b-7d74-43ab-8616-6044f625aef9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9913d namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: 55ada20d-8e14-4ef1-985a-5d0f54b55f24 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-9913d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9913d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:56:04Z" name: error-404-isvc-9913d.189dfddf6fa05af7 namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0023a7ff-aca5-43fd-9744-2b7194b93fde reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9913d namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: 55ada20d-8e14-4ef1-985a-5d0f54b55f24 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9913d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:56:04Z" name: error-404-isvc-9913d.189dfddf6fa09895 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 0d2dbfad-057a-4132-a1c8-1168e567f38b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9913d namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 55ada20d-8e14-4ef1-985a-5d0f54b55f24 kind: Event lastTimestamp: "2026-03-18T16:57:06Z" message: InferenceService [error-404-isvc-9913d] is Ready metadata: creationTimestamp: "2026-03-18T16:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:57:06Z" name: error-404-isvc-9913d.189dfdedde801cd9 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: 38b52036-85fe-4483-8f5b-1044668d3ebf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:53:44.049457Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn namespace: kserve-ci-e2e-test resourceVersion: "13536" uid: ebffde87-c362-47f0-b8dd-9798b092a274 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:53:44Z" name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn.189dfdbec54ab33f namespace: kserve-ci-e2e-test resourceVersion: "13540" uid: 5f9c9633-edc4-4c7c-ac8e-d9eb4e630163 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn namespace: kserve-ci-e2e-test resourceVersion: "13544" uid: ebffde87-c362-47f0-b8dd-9798b092a274 kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:53:44Z" name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn.189dfdbede2684e3 namespace: kserve-ci-e2e-test resourceVersion: "13606" uid: 9d94fee8-d7c5-42c6-8d9a-1c7f513d952a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn namespace: kserve-ci-e2e-test resourceVersion: "13537" uid: ebffde87-c362-47f0-b8dd-9798b092a274 kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:44Z" name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn.189dfdbedf462a36 namespace: kserve-ci-e2e-test resourceVersion: "13608" uid: 9aa09d01-b38b-413f-99f0-79609938812a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn namespace: kserve-ci-e2e-test resourceVersion: "13537" uid: ebffde87-c362-47f0-b8dd-9798b092a274 kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" in 6.436s (6.436s including waiting). Image size: 366724740 bytes.' metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn.189dfdc05eea8b06 namespace: kserve-ci-e2e-test resourceVersion: "13731" uid: c2798a11-0807-4167-a42d-0d05dd47f865 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn namespace: kserve-ci-e2e-test resourceVersion: "13537" uid: ebffde87-c362-47f0-b8dd-9798b092a274 kind: Event lastTimestamp: "2026-03-18T16:53:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T16:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:51Z" name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn.189dfdc063c08004 namespace: kserve-ci-e2e-test resourceVersion: "13734" uid: de4bf7c5-e349-4efd-9a66-341fe1ea6ef3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn namespace: kserve-ci-e2e-test resourceVersion: "13537" uid: ebffde87-c362-47f0-b8dd-9798b092a274 kind: Event lastTimestamp: "2026-03-18T16:53:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T16:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:51Z" name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn.189dfdc06432fc74 namespace: kserve-ci-e2e-test resourceVersion: "13736" uid: ce667ced-2a42-4b2f-98fb-db2132f0b543 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T16:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn namespace: kserve-ci-e2e-test resourceVersion: "13537" uid: ebffde87-c362-47f0-b8dd-9798b092a274 kind: Event lastTimestamp: "2026-03-18T16:54:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T16:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:42Z" name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn.189dfdc096fd61b3 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 128610c9-b323-4cd4-8f30-50ce40be7653 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn namespace: kserve-ci-e2e-test resourceVersion: "13537" uid: ebffde87-c362-47f0-b8dd-9798b092a274 kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn.189dfdd61b1a0811 namespace: kserve-ci-e2e-test resourceVersion: "14744" uid: 6ba038d5-df0a-449f-a544-3c1aa2ec761a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a225b-predictor-6f4b648d5b namespace: kserve-ci-e2e-test resourceVersion: "13529" uid: b8349f59-b25e-4eb5-a299-7d3f0c5fe47a kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: 'Created pod: error-404-isvc-a225b-predictor-6f4b648d5b-kq7qn' metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:44Z" name: error-404-isvc-a225b-predictor-6f4b648d5b.189dfdbec4fb453b namespace: kserve-ci-e2e-test resourceVersion: "13538" uid: 3a972f9f-2ea4-4517-8158-58b1768430ab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13528" uid: 741f82fe-3e5a-40c1-a26e-162462bd6be6 kind: Event lastTimestamp: "2026-03-18T16:53:43Z" message: Scaled up replica set error-404-isvc-a225b-predictor-6f4b648d5b from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:43Z" name: error-404-isvc-a225b-predictor.189dfdbec22bd4dc namespace: kserve-ci-e2e-test resourceVersion: "13531" uid: 851d5e73-36cc-4281-803a-54093cc169c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13530" uid: 70e968da-002d-4554-9d69-e1e9d8b2e0da kind: Event lastTimestamp: "2026-03-18T16:53:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:59Z" name: error-404-isvc-a225b-predictor.189dfdc2411c9a7c namespace: kserve-ci-e2e-test resourceVersion: "13862" uid: dea63b33-fc66-4cdf-ba21-db6526251397 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13530" uid: 70e968da-002d-4554-9d69-e1e9d8b2e0da kind: Event lastTimestamp: "2026-03-18T16:53:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:59Z" name: error-404-isvc-a225b-predictor.189dfdc2417fe1df namespace: kserve-ci-e2e-test resourceVersion: "13865" uid: d71fc99d-0a64-417c-8e1e-ad56e027a88f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T16:54:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13863" uid: 70e968da-002d-4554-9d69-e1e9d8b2e0da kind: Event lastTimestamp: "2026-03-18T16:55:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:14Z" name: error-404-isvc-a225b-predictor.189dfdc5c018809d namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: 31709c35-601d-4e0a-ae12-37823a6ec7b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T16:54:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13863" uid: 70e968da-002d-4554-9d69-e1e9d8b2e0da kind: Event lastTimestamp: "2026-03-18T16:55:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:14Z" name: error-404-isvc-a225b-predictor.189dfdc5c078cfcd namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: 10824f2e-0fab-471e-a914-171165020c38 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a225b namespace: kserve-ci-e2e-test resourceVersion: "13513" uid: 0e65f0d4-8f99-451a-baae-7ef41fb26fa1 kind: Event lastTimestamp: "2026-03-18T16:54:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:54:52Z" name: error-404-isvc-a225b.189dfdbec2616ca5 namespace: kserve-ci-e2e-test resourceVersion: "14339" uid: 70afb55a-9006-4dfe-882d-6973d2205e46 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a225b namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 0e65f0d4-8f99-451a-baae-7ef41fb26fa1 kind: Event lastTimestamp: "2026-03-18T16:54:52Z" message: InferenceService [error-404-isvc-a225b] is Ready metadata: creationTimestamp: "2026-03-18T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:54:52Z" name: error-404-isvc-a225b.189dfdceceaf8d80 namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: 9c5e8002-90d0-4c61-a49b-587a466753e0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:55:24.247491Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: 68676a2c-17bc-4b64-81d4-d23de1c32dd1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dfe46-predictor-d698cb6db-b6k9k to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k.189dfdd6198f557d namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: fe464d11-cd9c-437d-908b-512366977ad5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 68676a2c-17bc-4b64-81d4-d23de1c32dd1 kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k.189dfdd6327f1386 namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: c21fc0c4-ec12-4bb5-8dd4-ba8d6639b9c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 68676a2c-17bc-4b64-81d4-d23de1c32dd1 kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k.189dfdd63390dd84 namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 17de4ec0-ce3d-4fba-9205-906627626fda reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 68676a2c-17bc-4b64-81d4-d23de1c32dd1 kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k.189dfdd63ab2fd1e namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: 31367c3c-2bda-463e-90a0-af0e66b36bab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 68676a2c-17bc-4b64-81d4-d23de1c32dd1 kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k.189dfdd63b1d8d43 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: a9968eda-7875-4661-a1d3-966adaafd67c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T16:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 68676a2c-17bc-4b64-81d4-d23de1c32dd1 kind: Event lastTimestamp: "2026-03-18T16:56:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T16:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:16Z" name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k.189dfdd64f56b530 namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 4af4f44e-727f-4d34-b4c8-934d85fbd4b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 68676a2c-17bc-4b64-81d4-d23de1c32dd1 kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:04:59Z" name: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k.189dfe5c07f05723 namespace: kserve-ci-e2e-test resourceVersion: "18761" uid: 6ce5bbf2-d9d0-4df0-82fe-7fc0c2a7dfc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dfe46-predictor-d698cb6db namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: aa6082e6-e2d4-41e2-9bba-4bb5c88ae832 kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: 'Created pod: error-404-isvc-dfe46-predictor-d698cb6db-b6k9k' metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-dfe46-predictor-d698cb6db.189dfdd619197682 namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: c5cdfb8a-f5a9-4891-aa1c-ddded010b2c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dfe46-predictor namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: 56ac4778-bc15-4d68-b22a-6f92bd3ea80f kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: Scaled up replica set error-404-isvc-dfe46-predictor-d698cb6db from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-dfe46-predictor.189dfdd615235727 namespace: kserve-ci-e2e-test resourceVersion: "14717" uid: e8d6dded-55d4-4812-aafc-768bd9fbb6ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:55:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dfe46-predictor namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: a5e8365f-5854-4bb7-9790-4312dfbbfcd6 kind: Event lastTimestamp: "2026-03-18T16:56:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:09Z" name: error-404-isvc-dfe46-predictor.189dfdd993cf0226 namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 40007fc9-21e3-483a-915f-22cee9ff95c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:55:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dfe46-predictor namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: a5e8365f-5854-4bb7-9790-4312dfbbfcd6 kind: Event lastTimestamp: "2026-03-18T16:56:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:09Z" name: error-404-isvc-dfe46-predictor.189dfdd9942d3a6d namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 92ef6d9f-54e9-4d08-a22e-bccd2335d724 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:56:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dfe46-predictor namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: a5e8365f-5854-4bb7-9790-4312dfbbfcd6 kind: Event lastTimestamp: "2026-03-18T16:56:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:39Z" name: error-404-isvc-dfe46-predictor.189dfde4104068a2 namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: 00d97161-b938-4795-af7e-355ff1acedbb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:56:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dfe46-predictor namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: a5e8365f-5854-4bb7-9790-4312dfbbfcd6 kind: Event lastTimestamp: "2026-03-18T16:56:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:39Z" name: error-404-isvc-dfe46-predictor.189dfde4109f5fee namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 3dd39726-c4e8-46e5-97f5-febe7892e0a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 496e6a10-0649-4cfe-bedb-aef1340779aa kind: Event lastTimestamp: "2026-03-18T16:56:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:56:26Z" name: error-404-isvc-dfe46.189dfdd6153d41b9 namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: ad26dbba-14bd-47de-ad4e-3410d22af770 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 496e6a10-0649-4cfe-bedb-aef1340779aa kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: 'Failed to update status for InferenceService "error-404-isvc-dfe46": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dfe46": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-dfe46.189dfdd6184784f4 namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: 3451baf5-6293-4155-bba2-b8f682b87435 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 496e6a10-0649-4cfe-bedb-aef1340779aa kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dfe46": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:24Z" name: error-404-isvc-dfe46.189dfdd61847bac2 namespace: kserve-ci-e2e-test resourceVersion: "14723" uid: 66488c6f-5323-49cc-bc91-1d947d853720 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: 496e6a10-0649-4cfe-bedb-aef1340779aa kind: Event lastTimestamp: "2026-03-18T16:56:26Z" message: InferenceService [error-404-isvc-dfe46] is Ready metadata: creationTimestamp: "2026-03-18T16:56:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:56:26Z" name: error-404-isvc-dfe46.189dfde4874c0a4f namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: aa147a39-5e4c-4f4f-8a1f-fc2e705e1d5e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:16:44.404280Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-eb922-predictor-6b476995dd-bh64r namespace: kserve-ci-e2e-test resourceVersion: "22913" uid: 7c139f50-4575-43f9-b8ee-3b00be7c1b44 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-eb922-predictor-6b476995dd-bh64r to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-eb922-predictor-6b476995dd-bh64r.189dff0028d9c94c namespace: kserve-ci-e2e-test resourceVersion: "22917" uid: d36cd3f6-091a-4eea-a5e1-df30f0be556d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-eb922-predictor-6b476995dd-bh64r namespace: kserve-ci-e2e-test resourceVersion: "22922" uid: 7c139f50-4575-43f9-b8ee-3b00be7c1b44 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-eb922-predictor-6b476995dd-bh64r.189dff0042056458 namespace: kserve-ci-e2e-test resourceVersion: "22939" uid: 7d17be42-7768-4ae1-ae65-ee1eff98b712 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb922-predictor-6b476995dd-bh64r namespace: kserve-ci-e2e-test resourceVersion: "22915" uid: 7c139f50-4575-43f9-b8ee-3b00be7c1b44 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-eb922-predictor-6b476995dd-bh64r.189dff004316991a namespace: kserve-ci-e2e-test resourceVersion: "22941" uid: 224da229-04e3-4c7a-abb8-2701c1b3d9d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb922-predictor-6b476995dd-bh64r namespace: kserve-ci-e2e-test resourceVersion: "22915" uid: 7c139f50-4575-43f9-b8ee-3b00be7c1b44 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-eb922-predictor-6b476995dd-bh64r.189dff0046d4486a namespace: kserve-ci-e2e-test resourceVersion: "22947" uid: 2f06a543-d749-4802-b47a-ff515e56723b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb922-predictor-6b476995dd-bh64r namespace: kserve-ci-e2e-test resourceVersion: "22915" uid: 7c139f50-4575-43f9-b8ee-3b00be7c1b44 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-eb922-predictor-6b476995dd-bh64r.189dff004754e148 namespace: kserve-ci-e2e-test resourceVersion: "22948" uid: d570d20e-a9a2-43ff-b7d0-5e2d0b09e8fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb922-predictor-6b476995dd-bh64r namespace: kserve-ci-e2e-test resourceVersion: "22915" uid: 7c139f50-4575-43f9-b8ee-3b00be7c1b44 kind: Event lastTimestamp: "2026-03-18T17:34:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:34:25Z" name: error-404-isvc-eb922-predictor-6b476995dd-bh64r.189dff004b2223c8 namespace: kserve-ci-e2e-test resourceVersion: "28415" uid: 7ef83a62-65f7-43a0-b9ff-2251adb8ae18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:34:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb922-predictor-6b476995dd-bh64r namespace: kserve-ci-e2e-test resourceVersion: "22915" uid: 7c139f50-4575-43f9-b8ee-3b00be7c1b44 kind: Event lastTimestamp: "2026-03-18T17:34:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:34:23Z" name: error-404-isvc-eb922-predictor-6b476995dd-bh64r.189dfff6d1488be4 namespace: kserve-ci-e2e-test resourceVersion: "28370" uid: 0d90aa92-b407-48d0-be02-736f1d6b702e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-eb922-predictor-6b476995dd namespace: kserve-ci-e2e-test resourceVersion: "22905" uid: 4cc4fe79-9671-4a8d-bdf2-18f4c1cbeb4d kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: 'Created pod: error-404-isvc-eb922-predictor-6b476995dd-bh64r' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-eb922-predictor-6b476995dd.189dff00285f9e49 namespace: kserve-ci-e2e-test resourceVersion: "22914" uid: 9b7f8755-eedd-44b9-ad1f-ab21d8320d84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "22903" uid: 05603b77-01aa-48fa-9667-c169e94bd7d6 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: Scaled up replica set error-404-isvc-eb922-predictor-6b476995dd from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-eb922-predictor.189dff0027338ef0 namespace: kserve-ci-e2e-test resourceVersion: "22907" uid: 49d8e8e2-f48f-4615-97f4-14b9a4f14f0e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:16:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "22906" uid: 68be5b5c-1e8d-4f32-af20-3648051271c6 kind: Event lastTimestamp: "2026-03-18T17:17:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:14Z" name: error-404-isvc-eb922-predictor.189dff03a5e7b5a5 namespace: kserve-ci-e2e-test resourceVersion: "23309" uid: 2eb8d65c-a784-418b-8a34-25181116af18 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:16:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "22906" uid: 68be5b5c-1e8d-4f32-af20-3648051271c6 kind: Event lastTimestamp: "2026-03-18T17:17:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:14Z" name: error-404-isvc-eb922-predictor.189dff03a6423753 namespace: kserve-ci-e2e-test resourceVersion: "23310" uid: e941499b-fefa-4751-827a-80c33eac67a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:17:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "23041" uid: 68be5b5c-1e8d-4f32-af20-3648051271c6 kind: Event lastTimestamp: "2026-03-18T17:18:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:17:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:14Z" name: error-404-isvc-eb922-predictor.189dff0aa3c80390 namespace: kserve-ci-e2e-test resourceVersion: "23982" uid: 0d36d772-775e-4a2b-8638-0f4c30cdfcc7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:17:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "23041" uid: 68be5b5c-1e8d-4f32-af20-3648051271c6 kind: Event lastTimestamp: "2026-03-18T17:18:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:17:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:14Z" name: error-404-isvc-eb922-predictor.189dff0aa42655dc namespace: kserve-ci-e2e-test resourceVersion: "23983" uid: 2b9d2a92-dafe-41ea-aa61-72dd0aeca3fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eb922 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: cbd7ea6f-5a61-4238-84ac-be2b903adee7 kind: Event lastTimestamp: "2026-03-18T17:17:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:46Z" name: error-404-isvc-eb922.189dff00274ff5e5 namespace: kserve-ci-e2e-test resourceVersion: "23783" uid: 470e813a-1d7e-419e-b664-27079cccfd47 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eb922 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: cbd7ea6f-5a61-4238-84ac-be2b903adee7 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-eb922": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-eb922": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-eb922.189dff0027bfe8df namespace: kserve-ci-e2e-test resourceVersion: "22910" uid: cb60071f-9ffa-465a-b558-12d77b5bcdaa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eb922 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: cbd7ea6f-5a61-4238-84ac-be2b903adee7 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-eb922": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:16:44Z" name: error-404-isvc-eb922.189dff0027c01c93 namespace: kserve-ci-e2e-test resourceVersion: "22912" uid: 9cea5489-2aed-4db7-ab48-1a9a1208ca86 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eb922 namespace: kserve-ci-e2e-test resourceVersion: "23781" uid: cbd7ea6f-5a61-4238-84ac-be2b903adee7 kind: Event lastTimestamp: "2026-03-18T17:17:46Z" message: InferenceService [error-404-isvc-eb922] is Ready metadata: creationTimestamp: "2026-03-18T17:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:46Z" name: error-404-isvc-eb922.189dff0e82f458e1 namespace: kserve-ci-e2e-test resourceVersion: "23782" uid: 1f24254d-1284-433c-ab53-6b5559a984a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:53:43.772824Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13483" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6579795fb5-vvblb to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T16:53:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:53:43Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdbeb4cd9fc4 namespace: kserve-ci-e2e-test resourceVersion: "13488" uid: 6ae9ba18-fbcf-4798-a5cf-73f2472795ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13492" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:53:44Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdbecf877d60 namespace: kserve-ci-e2e-test resourceVersion: "13563" uid: 545d7297-bdd5-4fbd-b25a-1f9e77b8fa2e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:51226a883efdf7ccb64a3a56410fd7b7e354b39c67d730dd5df4437d36da2804" metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:44Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdbed1048221 namespace: kserve-ci-e2e-test resourceVersion: "13566" uid: ea072d16-dff9-4588-aed8-2103605d4b1c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:51226a883efdf7ccb64a3a56410fd7b7e354b39c67d730dd5df4437d36da2804" in 6.359s (6.359s including waiting). Image size: 298768935 bytes.' metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdc04c1118ca namespace: kserve-ci-e2e-test resourceVersion: "13716" uid: 50a67526-2459-4b13-bfc6-21f5d18e336d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdc04ff4a685 namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: 8a4d1b5b-7c66-4865-81c3-604823abd050 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdc0507dab7f namespace: kserve-ci-e2e-test resourceVersion: "13723" uid: 1ef9ca3c-e02d-4bc2-9560-4a46783f42ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:53:55Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1241" metadata: creationTimestamp: "2026-03-18T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:55Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdc167d79dba namespace: kserve-ci-e2e-test resourceVersion: "13806" uid: ba0ac294-5c6b-4f4b-b597-4ed78062fe6a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:54:00Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1241" in 5.421s (5.421s including waiting). Image size: 588839565 bytes.' metadata: creationTimestamp: "2026-03-18T16:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:00Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdc2aafdff73 namespace: kserve-ci-e2e-test resourceVersion: "13882" uid: 8d825354-05b4-4af6-a4f9-1cf07f63e282 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:54:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T16:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:00Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdc2ae121bc7 namespace: kserve-ci-e2e-test resourceVersion: "13884" uid: 45630fca-11f0-48cb-b946-41af1d1a655d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:54:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T16:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:00Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdc2ae9543b2 namespace: kserve-ci-e2e-test resourceVersion: "13886" uid: c6129fcc-fb5d-41e1-bab2-9c983a7e491b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-03-18T16:54:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:56:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T16:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:07Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfdc2cf18b6b0 namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: 0940f40e-f97c-4bf6-8a2b-2c223883d9c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: e9d24bd6-a499-475b-81ac-535409aeb9bf kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:04Z" name: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb.189dfddf6b658724 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: aec59c26-c47f-4f6a-8ac9-619b5f247ee5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6579795fb5 namespace: kserve-ci-e2e-test resourceVersion: "13472" uid: fe792989-c93b-4e38-b398-273a6321e230 kind: Event lastTimestamp: "2026-03-18T16:53:43Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6579795fb5-vvblb' metadata: creationTimestamp: "2026-03-18T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:43Z" name: isvc-sklearn-graph-1-predictor-6579795fb5.189dfdbeb45eeaf8 namespace: kserve-ci-e2e-test resourceVersion: "13484" uid: 1141b116-7972-4647-90e9-45fdb6f20621 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13470" uid: b98fae09-ce14-4617-a36e-cadfcc151e7d kind: Event lastTimestamp: "2026-03-18T16:53:43Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6579795fb5 from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:43Z" name: isvc-sklearn-graph-1-predictor.189dfdbead3e7d6f namespace: kserve-ci-e2e-test resourceVersion: "13473" uid: 32c77698-c291-4b08-af32-66865d4f613a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:53:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 947eacf8-9b46-449a-bd49-492d3536e0bd kind: Event lastTimestamp: "2026-03-18T16:54:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:54:28Z" name: isvc-sklearn-graph-1-predictor.189dfdc2322b59db namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: e17710af-eabd-40a0-b772-f92076485277 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:53:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 947eacf8-9b46-449a-bd49-492d3536e0bd kind: Event lastTimestamp: "2026-03-18T16:54:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:54:28Z" name: isvc-sklearn-graph-1-predictor.189dfdc2328d1599 namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: ee2173ff-98af-44f9-a956-263f7b3a63c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T16:54:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13856" uid: 947eacf8-9b46-449a-bd49-492d3536e0bd kind: Event lastTimestamp: "2026-03-18T16:55:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:58Z" name: isvc-sklearn-graph-1-predictor.189dfdccaf7aea92 namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: af35960e-bde4-4693-b031-43c0421aa85f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T16:54:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13856" uid: 947eacf8-9b46-449a-bd49-492d3536e0bd kind: Event lastTimestamp: "2026-03-18T16:55:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:58Z" name: isvc-sklearn-graph-1-predictor.189dfdccafe3c043 namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: 813ca162-f436-4377-8e40-2c05b88aa3b9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-03-18T16:53:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13459" uid: 6e3e4732-bd65-4318-8e8e-a395691579ca kind: Event lastTimestamp: "2026-03-18T16:55:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:27Z" name: isvc-sklearn-graph-1.189dfdbeb342ce68 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 78674003-de4d-4605-a8d5-0923332d9a03 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 6e3e4732-bd65-4318-8e8e-a395691579ca kind: Event lastTimestamp: "2026-03-18T16:55:27Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-03-18T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:27Z" name: isvc-sklearn-graph-1.189dfdd6e4b28fcd namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 55be3b89-3afc-4219-9d92-1d5a6062017f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:53:44.366874Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13590" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:53:44Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdbed8361e3b namespace: kserve-ci-e2e-test resourceVersion: "13596" uid: 8adec919-3ea1-45f9-a720-0d343dbc80a1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13600" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:53:44Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdbef25eab9c namespace: kserve-ci-e2e-test resourceVersion: "13620" uid: 4829225c-89ca-4e4d-b923-f2515f92e657 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:51226a883efdf7ccb64a3a56410fd7b7e354b39c67d730dd5df4437d36da2804" metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:44Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdbef3b475d9 namespace: kserve-ci-e2e-test resourceVersion: "13622" uid: 1951d555-2546-4e01-ad23-fd2f0e6b41d0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:51226a883efdf7ccb64a3a56410fd7b7e354b39c67d730dd5df4437d36da2804" in 5.778s (5.778s including waiting). Image size: 298768935 bytes.' metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdc04c1ada50 namespace: kserve-ci-e2e-test resourceVersion: "13717" uid: 9f1de9b5-8172-4c5b-a998-98e42077d363 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdc0506ee95a namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 265e2733-0e9a-4efc-a1cf-0aceea4c818e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdc050f18387 namespace: kserve-ci-e2e-test resourceVersion: "13724" uid: d8d09a6c-c92c-46af-9b9f-ab526b36ae6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:53:54Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1241" metadata: creationTimestamp: "2026-03-18T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:54Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdc12be6dcac namespace: kserve-ci-e2e-test resourceVersion: "13789" uid: b8d873b6-84b4-44dc-bcbe-645bf887d7f6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:54:00Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1241" in 6.412s (6.412s including waiting). Image size: 588839565 bytes.' metadata: creationTimestamp: "2026-03-18T16:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:00Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdc2aa17ce3c namespace: kserve-ci-e2e-test resourceVersion: "13881" uid: 9a76bc9a-85f0-44bf-ab98-440d59abbd85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:54:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T16:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:00Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdc2adfc9d8b namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: 55e46fb4-8d05-455c-921e-2e2592487b7c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:54:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T16:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:00Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdc2ae7a826e namespace: kserve-ci-e2e-test resourceVersion: "13885" uid: bc1debff-121d-4a2c-bbf9-0e1912a3798f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-03-18T16:54:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:56:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T16:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:08Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfdc2cf2bbf52 namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 1a8c291a-5124-46bf-8445-f0b0bd2a6d31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: 77931099-0e41-4ac7-81e1-41e67c4b0c9b kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:04Z" name: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg.189dfddf5fcd58e6 namespace: kserve-ci-e2e-test resourceVersion: "15225" uid: 3caec31e-9865-4912-b417-dcd1db02049a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-f89879bcc namespace: kserve-ci-e2e-test resourceVersion: "13583" uid: 28f06faf-689b-436b-9004-2dc0702b7bb5 kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-f89879bcc-6z6lg' metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:44Z" name: isvc-sklearn-graph-2-predictor-f89879bcc.189dfdbed7ca27f8 namespace: kserve-ci-e2e-test resourceVersion: "13591" uid: b28a5191-d210-41ff-a690-0a4c02a1cc50 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13581" uid: 8079debf-ae4f-4817-a318-b2693fe8c30e kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-f89879bcc from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:44Z" name: isvc-sklearn-graph-2-predictor.189dfdbed3c64823 namespace: kserve-ci-e2e-test resourceVersion: "13585" uid: 9e46f080-f3fc-45d7-be6f-60feca5423c9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:53:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13584" uid: 596a081a-7b80-4f46-8d77-362ddcc0717a kind: Event lastTimestamp: "2026-03-18T16:54:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:54:14Z" name: isvc-sklearn-graph-2-predictor.189dfdc252927807 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 11dce6b0-d8b6-4410-8c49-f77bc94c6137 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:53:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13584" uid: 596a081a-7b80-4f46-8d77-362ddcc0717a kind: Event lastTimestamp: "2026-03-18T16:54:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:54:14Z" name: isvc-sklearn-graph-2-predictor.189dfdc252fddace namespace: kserve-ci-e2e-test resourceVersion: "14016" uid: b1241049-6b2d-4127-8ef2-6771ee634652 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T16:54:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13872" uid: 596a081a-7b80-4f46-8d77-362ddcc0717a kind: Event lastTimestamp: "2026-03-18T16:55:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:44Z" name: isvc-sklearn-graph-2-predictor.189dfdc951656269 namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: 7361f18d-8465-49c6-b806-3e3507e16b1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T16:54:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13872" uid: 596a081a-7b80-4f46-8d77-362ddcc0717a kind: Event lastTimestamp: "2026-03-18T16:55:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:44Z" name: isvc-sklearn-graph-2-predictor.189dfdc952104869 namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 3252ca12-3511-47e6-b162-570d3f517d6c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13565" uid: 36df09ec-69c0-44a6-80c6-d9799f95f499 kind: Event lastTimestamp: "2026-03-18T16:55:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:28Z" name: isvc-sklearn-graph-2.189dfdbed5c559c6 namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 6cc36d19-75cc-4e7e-9b12-2ea57f785eb1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 36df09ec-69c0-44a6-80c6-d9799f95f499 kind: Event lastTimestamp: "2026-03-18T16:55:28Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-03-18T16:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:28Z" name: isvc-sklearn-graph-2.189dfdd725132efb namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: c498e320-2d00-4bdb-ac6c-0e0e65c5d867 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:53:44.263129Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13567" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-5c67486796-rk6k8 to ip-10-0-139-43.ec2.internal metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:53:44Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdbed2071886 namespace: kserve-ci-e2e-test resourceVersion: "13573" uid: 15d48d2f-ec23-4dc9-8dc2-9f536dc9817b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13582" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:53:44Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdbeead96330 namespace: kserve-ci-e2e-test resourceVersion: "13615" uid: c8064a74-b1d9-48a1-818e-df3d3e3f3f5f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:51226a883efdf7ccb64a3a56410fd7b7e354b39c67d730dd5df4437d36da2804" metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:44Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdbeecc93226 namespace: kserve-ci-e2e-test resourceVersion: "13619" uid: 68441ce4-7847-496a-a908-8ebb6471bae0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:51226a883efdf7ccb64a3a56410fd7b7e354b39c67d730dd5df4437d36da2804" in 6.183s (6.183s including waiting). Image size: 298768935 bytes.' metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdc05d5daa25 namespace: kserve-ci-e2e-test resourceVersion: "13730" uid: 18eac8d1-1a10-44c6-9532-aaa063f8e754 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:53:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-03-18T16:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:51Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdc063baffbb namespace: kserve-ci-e2e-test resourceVersion: "13733" uid: c7992696-fb09-4439-ac7b-4fa181c5b2be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:53:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-03-18T16:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:51Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdc064236b28 namespace: kserve-ci-e2e-test resourceVersion: "13735" uid: 58b79ad4-0508-40d1-a382-eef1a7aa7559 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:53:54Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-03-18T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:54Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdc14a6d6b93 namespace: kserve-ci-e2e-test resourceVersion: "13796" uid: b96734f0-4d5a-4807-9b44-c94e3d2df08e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:54:12Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.893s (17.893s including waiting). Image size: 1307475708 bytes.' metadata: creationTimestamp: "2026-03-18T16:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:12Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdc574fd0864 namespace: kserve-ci-e2e-test resourceVersion: "13987" uid: c9bfef7c-cb86-49d1-b794-2481f2d34ab7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:54:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T16:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:12Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdc5789f5c3d namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: bc286474-f47a-4a53-aabe-1302eeb6506d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:54:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T16:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:12Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdc5790523d9 namespace: kserve-ci-e2e-test resourceVersion: "13989" uid: f75c789f-7a84-4157-985f-e3de3dc430b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T16:54:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:55:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T16:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:03Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfdc57f2d6a4d namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 3cb30fd8-aba4-4df6-86cc-372189dafbd8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-5c67486796-rk6k8 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: 44daecb7-0fa2-4687-9d98-422f0cc3a248 kind: Event lastTimestamp: "2026-03-18T16:56:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T16:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:03Z" name: isvc-xgboost-graph-predictor-5c67486796-rk6k8.189dfddf59392025 namespace: kserve-ci-e2e-test resourceVersion: "15196" uid: 64904f91-8cd0-44ff-91ab-7eb763b4badc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-43.ec2.internal source: component: kubelet host: ip-10-0-139-43.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-5c67486796 namespace: kserve-ci-e2e-test resourceVersion: "13557" uid: a117c100-ab3c-47d2-92a0-18ba9e78f20d kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: 'Created pod: isvc-xgboost-graph-predictor-5c67486796-rk6k8' metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:44Z" name: isvc-xgboost-graph-predictor-5c67486796.189dfdbed16dfa23 namespace: kserve-ci-e2e-test resourceVersion: "13569" uid: 4a1215ac-3e6c-460c-babe-77d5cad7e723 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13555" uid: d0423542-7746-461f-bc94-76014b5c3e33 kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Scaled up replica set isvc-xgboost-graph-predictor-5c67486796 from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:44Z" name: isvc-xgboost-graph-predictor.189dfdbeccd7ab37 namespace: kserve-ci-e2e-test resourceVersion: "13559" uid: fc70c76e-0e97-4ce2-b1a1-6102f283dceb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:53:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13556" uid: 65d34e90-c79f-4029-8b16-3a3d7282edb3 kind: Event lastTimestamp: "2026-03-18T16:54:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:54:29Z" name: isvc-xgboost-graph-predictor.189dfdc24ba0a5e2 namespace: kserve-ci-e2e-test resourceVersion: "14135" uid: 0a843694-9fb9-4ac1-8c2d-fa3346f24a46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:53:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13556" uid: 65d34e90-c79f-4029-8b16-3a3d7282edb3 kind: Event lastTimestamp: "2026-03-18T16:54:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:54:29Z" name: isvc-xgboost-graph-predictor.189dfdc24c3437b6 namespace: kserve-ci-e2e-test resourceVersion: "14136" uid: 2916664d-82ce-43da-b5fc-fce6aa6ddd4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T16:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13868" uid: 65d34e90-c79f-4029-8b16-3a3d7282edb3 kind: Event lastTimestamp: "2026-03-18T16:55:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:29Z" name: isvc-xgboost-graph-predictor.189dfdccc867b8af namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 0cbf6efc-7207-417c-88f4-8d960d759837 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T16:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13868" uid: 65d34e90-c79f-4029-8b16-3a3d7282edb3 kind: Event lastTimestamp: "2026-03-18T16:55:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:29Z" name: isvc-xgboost-graph-predictor.189dfdccc8d25b61 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 7c2866cf-bfdf-4a2f-9206-4bc93f0e1678 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13545" uid: 5d82e538-9eff-47b7-87e6-ccad067375a8 kind: Event lastTimestamp: "2026-03-18T16:55:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:14Z" name: isvc-xgboost-graph.189dfdbeccb60c1f namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 4501fdfa-59e7-4991-9dc7-8fabd859f56c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: 5d82e538-9eff-47b7-87e6-ccad067375a8 kind: Event lastTimestamp: "2026-03-18T16:55:14Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-03-18T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:14Z" name: isvc-xgboost-graph.189dfdd3b799ea60 namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: b29167ad-6a6d-49cf-a9f1-0ed95a02c401 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:55:53.704134Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5894969db4-z496p namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: ee702450-de44-4b59-810b-1b89e630f1ab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5894969db4-z496p to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T16:55:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:55:53Z" name: model-chainer-5894969db4-z496p.189dfddcf55007e2 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: 09281b27-d3f1-4ffe-a7c7-4684085d3d78 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:54Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5894969db4-z496p namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: ee702450-de44-4b59-810b-1b89e630f1ab kind: Event lastTimestamp: "2026-03-18T16:55:54Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:55:54Z" name: model-chainer-5894969db4-z496p.189dfddd0f005166 namespace: kserve-ci-e2e-test resourceVersion: "15071" uid: 4127932c-b9f1-4e47-a014-1a1881e52f2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894969db4-z496p namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: ee702450-de44-4b59-810b-1b89e630f1ab kind: Event lastTimestamp: "2026-03-18T16:55:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T16:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:54Z" name: model-chainer-5894969db4-z496p.189dfddd103aacf0 namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: a1a0a262-18b3-431f-a7d3-272357fecc06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894969db4-z496p namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: ee702450-de44-4b59-810b-1b89e630f1ab kind: Event lastTimestamp: "2026-03-18T16:55:54Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-03-18T16:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:54Z" name: model-chainer-5894969db4-z496p.189dfddd135f2723 namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: 18b43fde-02fe-4797-a038-19deb7c0a591 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894969db4-z496p namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: ee702450-de44-4b59-810b-1b89e630f1ab kind: Event lastTimestamp: "2026-03-18T16:55:54Z" message: Started container model-chainer metadata: creationTimestamp: "2026-03-18T16:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:54Z" name: model-chainer-5894969db4-z496p.189dfddd13e51d54 namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: 4ac2432c-bcf2-4018-a6f8-4e8c79729832 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894969db4-z496p namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: ee702450-de44-4b59-810b-1b89e630f1ab kind: Event lastTimestamp: "2026-03-18T16:56:03Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-03-18T16:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:03Z" name: model-chainer-5894969db4-z496p.189dfddf4e7e5973 namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: c1e48698-a02c-4ada-a108-b5760c90a7ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T16:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5894969db4-z496p namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: ee702450-de44-4b59-810b-1b89e630f1ab kind: Event lastTimestamp: "2026-03-18T16:56:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T16:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:30Z" name: model-chainer-5894969db4-z496p.189dfddfc6037d8d namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 50e987b7-bbf5-4dcb-84dc-b9c036df5eef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5894969db4 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 1c0b3bd9-af2a-474d-9cc4-3a312098d4fb kind: Event lastTimestamp: "2026-03-18T16:55:53Z" message: 'Created pod: model-chainer-5894969db4-z496p' metadata: creationTimestamp: "2026-03-18T16:55:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:53Z" name: model-chainer-5894969db4.189dfddcf4fcc67e namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 7fdc1370-972d-49d5-a24d-04fbe34586d2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 2b1674f8-4e7a-451f-8449-2c9d0b6d313d kind: Event lastTimestamp: "2026-03-18T16:55:53Z" message: Scaled up replica set model-chainer-5894969db4 from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:55:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:53Z" name: model-chainer.189dfddcf43cbb95 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 7d65f969-efb3-46a4-949c-5dbe2bbc7e39 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 39c215ed-a328-4b1d-ab85-a973875826f4 kind: Event lastTimestamp: "2026-03-18T16:55:53Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-03-18T16:55:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:53Z" name: model-chainer.189dfddcf7c17357 namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: d92348a0-6078-4294-ba0d-3e91850645bb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:06:58.865286Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-18dc8-546dc7d746-98whx namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: 603b043e-b360-43a4-9780-75059a19a711 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-18dc8-546dc7d746-98whx to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:06:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:06:58Z" name: sequence-graph-18dc8-546dc7d746-98whx.189dfe77d4025b1a namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: 7780678e-efeb-4a47-8e9b-a5dbdff240b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:59Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-18dc8-546dc7d746-98whx namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: 603b043e-b360-43a4-9780-75059a19a711 kind: Event lastTimestamp: "2026-03-18T17:06:59Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:06:59Z" name: sequence-graph-18dc8-546dc7d746-98whx.189dfe77ed485692 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 0b543e8b-6ac6-4c50-8c8e-f2ac817bd780 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-18dc8} kind: Pod name: sequence-graph-18dc8-546dc7d746-98whx namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 603b043e-b360-43a4-9780-75059a19a711 kind: Event lastTimestamp: "2026-03-18T17:06:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T17:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:59Z" name: sequence-graph-18dc8-546dc7d746-98whx.189dfe77eefe09a1 namespace: kserve-ci-e2e-test resourceVersion: "19782" uid: c7e7a02d-ab29-46d3-b5b1-e1bddf1986f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-18dc8} kind: Pod name: sequence-graph-18dc8-546dc7d746-98whx namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 603b043e-b360-43a4-9780-75059a19a711 kind: Event lastTimestamp: "2026-03-18T17:06:59Z" message: 'Created container: sequence-graph-18dc8' metadata: creationTimestamp: "2026-03-18T17:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:59Z" name: sequence-graph-18dc8-546dc7d746-98whx.189dfe77f31cbb9d namespace: kserve-ci-e2e-test resourceVersion: "19783" uid: d6b68bfb-3391-4fab-b9c8-b9f9059edfa4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-18dc8} kind: Pod name: sequence-graph-18dc8-546dc7d746-98whx namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 603b043e-b360-43a4-9780-75059a19a711 kind: Event lastTimestamp: "2026-03-18T17:06:59Z" message: Started container sequence-graph-18dc8 metadata: creationTimestamp: "2026-03-18T17:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:59Z" name: sequence-graph-18dc8-546dc7d746-98whx.189dfe77f395962e namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: d6db555b-6ed7-4334-b7a4-8b70301b5c1c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-18dc8} kind: Pod name: sequence-graph-18dc8-546dc7d746-98whx namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 603b043e-b360-43a4-9780-75059a19a711 kind: Event lastTimestamp: "2026-03-18T17:07:08Z" message: Stopping container sequence-graph-18dc8 metadata: creationTimestamp: "2026-03-18T17:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:08Z" name: sequence-graph-18dc8-546dc7d746-98whx.189dfe7a2c5f8791 namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: fdbf6fe3-1b65-4e09-b967-d120dbb43923 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-18dc8} kind: Pod name: sequence-graph-18dc8-546dc7d746-98whx namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 603b043e-b360-43a4-9780-75059a19a711 kind: Event lastTimestamp: "2026-03-18T17:07:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T17:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:35Z" name: sequence-graph-18dc8-546dc7d746-98whx.189dfe7aa5af10c9 namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 0928cefa-36e9-404c-b3d4-dde2d2e2b867 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-18dc8-546dc7d746 namespace: kserve-ci-e2e-test resourceVersion: "19758" uid: 41f7caa3-7dff-436d-8dba-ca4d1e5b1e01 kind: Event lastTimestamp: "2026-03-18T17:06:58Z" message: 'Created pod: sequence-graph-18dc8-546dc7d746-98whx' metadata: creationTimestamp: "2026-03-18T17:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:58Z" name: sequence-graph-18dc8-546dc7d746.189dfe77d39e6e82 namespace: kserve-ci-e2e-test resourceVersion: "19763" uid: d6900761-2143-41c9-8f73-159e94ed164e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-18dc8 namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: f71f955c-e8a8-4714-856c-6dcb9fe3c96a kind: Event lastTimestamp: "2026-03-18T17:06:58Z" message: Scaled up replica set sequence-graph-18dc8-546dc7d746 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:58Z" name: sequence-graph-18dc8.189dfe77d2daadb3 namespace: kserve-ci-e2e-test resourceVersion: "19760" uid: b1b2769b-4b0a-4ff4-932f-4cae9cfa2b24 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:58Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-18dc8 namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: 6fa948fe-78af-4f8f-8e99-f855a73dac27 kind: Event lastTimestamp: "2026-03-18T17:06:58Z" message: InferenceGraph [sequence-graph-18dc8] is Ready metadata: creationTimestamp: "2026-03-18T17:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:58Z" name: sequence-graph-18dc8.189dfe77d69753f6 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 9cc865bb-108d-454f-9f1e-9dc980f16ab0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:08:29.286994Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d83a-775c8cf4fc-289tk namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 48e1881b-e2d9-4786-8624-1d663c24a219 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8d83a-775c8cf4fc-289tk to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:08:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:08:29Z" name: sequence-graph-8d83a-775c8cf4fc-289tk.189dfe8ce18f8362 namespace: kserve-ci-e2e-test resourceVersion: "20501" uid: 2313adfe-3b1b-4e85-955e-af8837ad0192 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d83a-775c8cf4fc-289tk namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 48e1881b-e2d9-4786-8624-1d663c24a219 kind: Event lastTimestamp: "2026-03-18T17:08:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-8d83a-serving-cert" not found' metadata: creationTimestamp: "2026-03-18T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:08:29Z" name: sequence-graph-8d83a-775c8cf4fc-289tk.189dfe8cf0b5122d namespace: kserve-ci-e2e-test resourceVersion: "20512" uid: 7e1ad8fc-fb03-4af8-8dfa-0468a4fed673 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d83a-775c8cf4fc-289tk namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: 48e1881b-e2d9-4786-8624-1d663c24a219 kind: Event lastTimestamp: "2026-03-18T17:08:30Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:08:30Z" name: sequence-graph-8d83a-775c8cf4fc-289tk.189dfe8d1eab996d namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: a5d847f6-c0d2-40d6-852d-97f33685e60b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d83a} kind: Pod name: sequence-graph-8d83a-775c8cf4fc-289tk namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 48e1881b-e2d9-4786-8624-1d663c24a219 kind: Event lastTimestamp: "2026-03-18T17:08:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T17:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:08:30Z" name: sequence-graph-8d83a-775c8cf4fc-289tk.189dfe8d2017edd1 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: efbad4f5-7889-48cf-be56-1ff44ceb9990 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d83a} kind: Pod name: sequence-graph-8d83a-775c8cf4fc-289tk namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 48e1881b-e2d9-4786-8624-1d663c24a219 kind: Event lastTimestamp: "2026-03-18T17:08:30Z" message: 'Created container: sequence-graph-8d83a' metadata: creationTimestamp: "2026-03-18T17:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:08:30Z" name: sequence-graph-8d83a-775c8cf4fc-289tk.189dfe8d236f42a5 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 1b042965-83d5-497f-80cf-4f185a05990e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d83a} kind: Pod name: sequence-graph-8d83a-775c8cf4fc-289tk namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 48e1881b-e2d9-4786-8624-1d663c24a219 kind: Event lastTimestamp: "2026-03-18T17:08:30Z" message: Started container sequence-graph-8d83a metadata: creationTimestamp: "2026-03-18T17:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:08:30Z" name: sequence-graph-8d83a-775c8cf4fc-289tk.189dfe8d23e53844 namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 93f5b990-e808-4e4f-917f-a355f83f466b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d83a} kind: Pod name: sequence-graph-8d83a-775c8cf4fc-289tk namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 48e1881b-e2d9-4786-8624-1d663c24a219 kind: Event lastTimestamp: "2026-03-18T17:16:43Z" message: Stopping container sequence-graph-8d83a metadata: creationTimestamp: "2026-03-18T17:16:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:43Z" name: sequence-graph-8d83a-775c8cf4fc-289tk.189dff00084fcd9e namespace: kserve-ci-e2e-test resourceVersion: "22841" uid: 98f6c8a2-b0aa-48fe-8351-3ed05330d829 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d83a} kind: Pod name: sequence-graph-8d83a-775c8cf4fc-289tk namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 48e1881b-e2d9-4786-8624-1d663c24a219 kind: Event lastTimestamp: "2026-03-18T17:17:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T17:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:12Z" name: sequence-graph-8d83a-775c8cf4fc-289tk.189dff00d47bae7d namespace: kserve-ci-e2e-test resourceVersion: "23129" uid: b2dc60f5-7487-476a-9e41-75546ffd8f0b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8d83a-775c8cf4fc namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 33a4592c-361f-4059-951b-37356932da2b kind: Event lastTimestamp: "2026-03-18T17:08:29Z" message: 'Created pod: sequence-graph-8d83a-775c8cf4fc-289tk' metadata: creationTimestamp: "2026-03-18T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:29Z" name: sequence-graph-8d83a-775c8cf4fc.189dfe8ce12b80c2 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: ea7a0839-30a5-440e-a970-7981c97c8603 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8d83a namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: da4ffc92-b4ca-4cc3-9ad2-15c69a8f8279 kind: Event lastTimestamp: "2026-03-18T17:08:29Z" message: Scaled up replica set sequence-graph-8d83a-775c8cf4fc from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:29Z" name: sequence-graph-8d83a.189dfe8ce0751452 namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: 8fb4d4d0-5ead-448b-97c7-f014d26199d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8d83a namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: d95cfc53-21c4-4aee-8e83-a989049e6d8d kind: Event lastTimestamp: "2026-03-18T17:08:29Z" message: InferenceGraph [sequence-graph-8d83a] is Ready metadata: creationTimestamp: "2026-03-18T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:08:29Z" name: sequence-graph-8d83a.189dfe8ce3c2228c namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: 751dfe72-f8d9-45dc-90c9-d1231880cc8c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:08:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d83a namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 487f0b9c-4221-4bc0-bc0f-fab0a8f0490b kind: Event lastTimestamp: "2026-03-18T17:08:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:59Z" name: sequence-graph-8d83a.189dfe905ee9aaad namespace: kserve-ci-e2e-test resourceVersion: "20665" uid: 8f6e1dc5-5f75-42d5-a82b-418f7f907d42 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:08:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d83a namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 487f0b9c-4221-4bc0-bc0f-fab0a8f0490b kind: Event lastTimestamp: "2026-03-18T17:08:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:59Z" name: sequence-graph-8d83a.189dfe905f4de87b namespace: kserve-ci-e2e-test resourceVersion: "20666" uid: 7b97a6ab-f448-47cb-8066-1ae3386e4d63 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:57:23.989808Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9913d-8999cff78-v8r6t namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 16f2220f-605e-47aa-ba58-5f1f72775f0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-9913d-8999cff78-v8r6t to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T16:57:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:57:23Z" name: sequence-graph-9913d-8999cff78-v8r6t.189dfdf1fac21905 namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: b7a6f4f5-44b6-48af-afbf-f761b50679ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:24Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9913d-8999cff78-v8r6t namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 16f2220f-605e-47aa-ba58-5f1f72775f0e kind: Event lastTimestamp: "2026-03-18T16:57:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-9913d-serving-cert" not found' metadata: creationTimestamp: "2026-03-18T16:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:57:24Z" name: sequence-graph-9913d-8999cff78-v8r6t.189dfdf202d4cf64 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: baa5dc9a-9f48-4344-a98f-1ff7fa0cf66b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:25Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9913d-8999cff78-v8r6t namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 16f2220f-605e-47aa-ba58-5f1f72775f0e kind: Event lastTimestamp: "2026-03-18T16:57:25Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:57:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:57:25Z" name: sequence-graph-9913d-8999cff78-v8r6t.189dfdf2379306ce namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 9c34ca33-258e-4018-a6b1-717f8348a996 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9913d} kind: Pod name: sequence-graph-9913d-8999cff78-v8r6t namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 16f2220f-605e-47aa-ba58-5f1f72775f0e kind: Event lastTimestamp: "2026-03-18T16:57:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T16:57:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:57:25Z" name: sequence-graph-9913d-8999cff78-v8r6t.189dfdf238d7946f namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 8834307c-8837-46d8-8ce0-e558a72b901d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9913d} kind: Pod name: sequence-graph-9913d-8999cff78-v8r6t namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 16f2220f-605e-47aa-ba58-5f1f72775f0e kind: Event lastTimestamp: "2026-03-18T16:57:25Z" message: 'Created container: sequence-graph-9913d' metadata: creationTimestamp: "2026-03-18T16:57:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:57:25Z" name: sequence-graph-9913d-8999cff78-v8r6t.189dfdf23c3b6e14 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: af3430d4-0e8d-4119-b242-777fb8cefa40 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9913d} kind: Pod name: sequence-graph-9913d-8999cff78-v8r6t namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 16f2220f-605e-47aa-ba58-5f1f72775f0e kind: Event lastTimestamp: "2026-03-18T16:57:25Z" message: Started container sequence-graph-9913d metadata: creationTimestamp: "2026-03-18T16:57:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:57:25Z" name: sequence-graph-9913d-8999cff78-v8r6t.189dfdf23cb22033 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: ef7820d6-85b2-47f6-a29d-e51b1f08eae8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9913d} kind: Pod name: sequence-graph-9913d-8999cff78-v8r6t namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 16f2220f-605e-47aa-ba58-5f1f72775f0e kind: Event lastTimestamp: "2026-03-18T17:05:38Z" message: Stopping container sequence-graph-9913d metadata: creationTimestamp: "2026-03-18T17:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:38Z" name: sequence-graph-9913d-8999cff78-v8r6t.189dfe6525a6171c namespace: kserve-ci-e2e-test resourceVersion: "19003" uid: ec6c4d3c-265e-443e-93a9-5438bdf712a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9913d} kind: Pod name: sequence-graph-9913d-8999cff78-v8r6t namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 16f2220f-605e-47aa-ba58-5f1f72775f0e kind: Event lastTimestamp: "2026-03-18T17:06:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T17:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:07Z" name: sequence-graph-9913d-8999cff78-v8r6t.189dfe65f4f79fca namespace: kserve-ci-e2e-test resourceVersion: "19282" uid: c5ba86a0-78fe-4c4d-83ee-babf470c9dd5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-9913d-8999cff78 namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 57e5db5f-9692-4416-a0f3-c5cea3968ca2 kind: Event lastTimestamp: "2026-03-18T16:57:23Z" message: 'Created pod: sequence-graph-9913d-8999cff78-v8r6t' metadata: creationTimestamp: "2026-03-18T16:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:23Z" name: sequence-graph-9913d-8999cff78.189dfdf1fa138993 namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 5a912eda-2d17-4907-a763-a3a5cd1f0f4a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-9913d namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e3bac8b6-7d8d-48b4-a6d3-5edc7e8a82c3 kind: Event lastTimestamp: "2026-03-18T16:57:23Z" message: Scaled up replica set sequence-graph-9913d-8999cff78 from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:23Z" name: sequence-graph-9913d.189dfdf1f9286a4e namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 30c2e8c0-13f8-4e8d-82fb-4e24c2eb1b91 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9913d namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: e126652b-93ad-4052-8a8c-f22e4d0eff6f kind: Event lastTimestamp: "2026-03-18T16:57:24Z" message: InferenceGraph [sequence-graph-9913d] is Ready metadata: creationTimestamp: "2026-03-18T16:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:57:24Z" name: sequence-graph-9913d.189dfdf1fef5de88 namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 9dee4075-a1e5-4abe-9207-a8b23c96a4e4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9913d namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: ae8df812-f63a-43a9-b4d1-5c6408bb3319 kind: Event lastTimestamp: "2026-03-18T16:57:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:53Z" name: sequence-graph-9913d.189dfdf577ef767a namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 30439451-1ae7-4874-aa07-ec1f2c54794a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9913d namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: ae8df812-f63a-43a9-b4d1-5c6408bb3319 kind: Event lastTimestamp: "2026-03-18T16:57:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:53Z" name: sequence-graph-9913d.189dfdf5784d6808 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: ee964d2b-b027-455d-860b-94ca73e351eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:17:24.526180Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-0c4f8-7c56559465-2s29v namespace: kserve-ci-e2e-test resourceVersion: "23489" uid: 6557ac99-6842-4706-83ab-858761eb5031 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-0c4f8-7c56559465-2s29v to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:17:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:17:24Z" name: splitter-graph-0c4f8-7c56559465-2s29v.189dff09804d67e0 namespace: kserve-ci-e2e-test resourceVersion: "23494" uid: 3e388450-1342-422d-a7f2-a09428b1b136 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:24Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-0c4f8-7c56559465-2s29v namespace: kserve-ci-e2e-test resourceVersion: "23498" uid: 6557ac99-6842-4706-83ab-858761eb5031 kind: Event lastTimestamp: "2026-03-18T17:17:24Z" message: Add eth0 [10.133.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:17:24Z" name: splitter-graph-0c4f8-7c56559465-2s29v.189dff0999918166 namespace: kserve-ci-e2e-test resourceVersion: "23507" uid: 61184127-a79e-4055-98ea-dd2ebd86a281 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c4f8} kind: Pod name: splitter-graph-0c4f8-7c56559465-2s29v namespace: kserve-ci-e2e-test resourceVersion: "23491" uid: 6557ac99-6842-4706-83ab-858761eb5031 kind: Event lastTimestamp: "2026-03-18T17:17:24Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T17:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:24Z" name: splitter-graph-0c4f8-7c56559465-2s29v.189dff099acace44 namespace: kserve-ci-e2e-test resourceVersion: "23509" uid: aeeb2fa9-507a-4227-9c5a-d86eb780ad04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c4f8} kind: Pod name: splitter-graph-0c4f8-7c56559465-2s29v namespace: kserve-ci-e2e-test resourceVersion: "23491" uid: 6557ac99-6842-4706-83ab-858761eb5031 kind: Event lastTimestamp: "2026-03-18T17:17:25Z" message: 'Created container: splitter-graph-0c4f8' metadata: creationTimestamp: "2026-03-18T17:17:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:25Z" name: splitter-graph-0c4f8-7c56559465-2s29v.189dff099e255473 namespace: kserve-ci-e2e-test resourceVersion: "23511" uid: 533e477e-21ce-4ed7-bb2b-0980481e41b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c4f8} kind: Pod name: splitter-graph-0c4f8-7c56559465-2s29v namespace: kserve-ci-e2e-test resourceVersion: "23491" uid: 6557ac99-6842-4706-83ab-858761eb5031 kind: Event lastTimestamp: "2026-03-18T17:17:25Z" message: Started container splitter-graph-0c4f8 metadata: creationTimestamp: "2026-03-18T17:17:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:25Z" name: splitter-graph-0c4f8-7c56559465-2s29v.189dff099e983c55 namespace: kserve-ci-e2e-test resourceVersion: "23512" uid: c89fed9c-658c-47cf-963a-a609d01c188e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c4f8} kind: Pod name: splitter-graph-0c4f8-7c56559465-2s29v namespace: kserve-ci-e2e-test resourceVersion: "23491" uid: 6557ac99-6842-4706-83ab-858761eb5031 kind: Event lastTimestamp: "2026-03-18T17:17:34Z" message: Stopping container splitter-graph-0c4f8 metadata: creationTimestamp: "2026-03-18T17:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:34Z" name: splitter-graph-0c4f8-7c56559465-2s29v.189dff0bd8630725 namespace: kserve-ci-e2e-test resourceVersion: "23587" uid: 152c2960-721a-400a-bddf-9e2467c65ad6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:17:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-0c4f8} kind: Pod name: splitter-graph-0c4f8-7c56559465-2s29v namespace: kserve-ci-e2e-test resourceVersion: "23491" uid: 6557ac99-6842-4706-83ab-858761eb5031 kind: Event lastTimestamp: "2026-03-18T17:18:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T17:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:02Z" name: splitter-graph-0c4f8-7c56559465-2s29v.189dff0c68a3d7b8 namespace: kserve-ci-e2e-test resourceVersion: "23868" uid: 673aee82-890b-484f-a4f2-77c79b08789a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-0c4f8-7c56559465 namespace: kserve-ci-e2e-test resourceVersion: "23486" uid: 00104960-e509-4c25-9c13-a09adda6005e kind: Event lastTimestamp: "2026-03-18T17:17:24Z" message: 'Created pod: splitter-graph-0c4f8-7c56559465-2s29v' metadata: creationTimestamp: "2026-03-18T17:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:24Z" name: splitter-graph-0c4f8-7c56559465.189dff097fcad6aa namespace: kserve-ci-e2e-test resourceVersion: "23490" uid: 640d9973-1c1b-418e-82a0-bdd9b13c4b9c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "23484" uid: 908be784-4c61-41e1-a77d-df913b3b9bff kind: Event lastTimestamp: "2026-03-18T17:17:24Z" message: Scaled up replica set splitter-graph-0c4f8-7c56559465 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:24Z" name: splitter-graph-0c4f8.189dff097f232e3e namespace: kserve-ci-e2e-test resourceVersion: "23487" uid: d3dba1d1-1c3e-42c4-b877-062b0096b71e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "23500" uid: 3e3ad5ef-e683-4770-a81a-aa3d895dac83 kind: Event lastTimestamp: "2026-03-18T17:17:24Z" message: InferenceGraph [splitter-graph-0c4f8] is Ready metadata: creationTimestamp: "2026-03-18T17:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:24Z" name: splitter-graph-0c4f8.189dff09827b5f50 namespace: kserve-ci-e2e-test resourceVersion: "23502" uid: bb4fadf4-e8b9-45c8-9627-46cb137b230e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:18:55.104233Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6eae8-7cdb674958-h2pjc namespace: kserve-ci-e2e-test resourceVersion: "24213" uid: 08be6d3f-7873-46f4-a259-a0e4467eca34 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-6eae8-7cdb674958-h2pjc to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:18:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:18:55Z" name: splitter-graph-6eae8-7cdb674958-h2pjc.189dff1e972ccbfd namespace: kserve-ci-e2e-test resourceVersion: "24218" uid: ade0884b-7479-40b3-8293-31ffd98818f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:55Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6eae8-7cdb674958-h2pjc namespace: kserve-ci-e2e-test resourceVersion: "24215" uid: 08be6d3f-7873-46f4-a259-a0e4467eca34 kind: Event lastTimestamp: "2026-03-18T17:18:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-6eae8-serving-cert" not found' metadata: creationTimestamp: "2026-03-18T17:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:55Z" name: splitter-graph-6eae8-7cdb674958-h2pjc.189dff1ea2bfed27 namespace: kserve-ci-e2e-test resourceVersion: "24229" uid: e1c8c68a-de8c-49ed-98da-77533eaee928 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:56Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6eae8-7cdb674958-h2pjc namespace: kserve-ci-e2e-test resourceVersion: "24221" uid: 08be6d3f-7873-46f4-a259-a0e4467eca34 kind: Event lastTimestamp: "2026-03-18T17:18:56Z" message: Add eth0 [10.133.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:18:56Z" name: splitter-graph-6eae8-7cdb674958-h2pjc.189dff1ed42a0818 namespace: kserve-ci-e2e-test resourceVersion: "24236" uid: 2ea1eb67-4686-48ea-8cbd-1bd66bf3c3b3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6eae8} kind: Pod name: splitter-graph-6eae8-7cdb674958-h2pjc namespace: kserve-ci-e2e-test resourceVersion: "24215" uid: 08be6d3f-7873-46f4-a259-a0e4467eca34 kind: Event lastTimestamp: "2026-03-18T17:18:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T17:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:56Z" name: splitter-graph-6eae8-7cdb674958-h2pjc.189dff1ed5d0a49c namespace: kserve-ci-e2e-test resourceVersion: "24238" uid: 405bb5e0-70fb-4cd0-8353-a2f1a8eab63a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6eae8} kind: Pod name: splitter-graph-6eae8-7cdb674958-h2pjc namespace: kserve-ci-e2e-test resourceVersion: "24215" uid: 08be6d3f-7873-46f4-a259-a0e4467eca34 kind: Event lastTimestamp: "2026-03-18T17:18:56Z" message: 'Created container: splitter-graph-6eae8' metadata: creationTimestamp: "2026-03-18T17:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:56Z" name: splitter-graph-6eae8-7cdb674958-h2pjc.189dff1ed8e8ee2d namespace: kserve-ci-e2e-test resourceVersion: "24240" uid: 364afef3-30d6-49db-a23f-e0b7114d2c0e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6eae8} kind: Pod name: splitter-graph-6eae8-7cdb674958-h2pjc namespace: kserve-ci-e2e-test resourceVersion: "24215" uid: 08be6d3f-7873-46f4-a259-a0e4467eca34 kind: Event lastTimestamp: "2026-03-18T17:18:56Z" message: Started container splitter-graph-6eae8 metadata: creationTimestamp: "2026-03-18T17:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:56Z" name: splitter-graph-6eae8-7cdb674958-h2pjc.189dff1ed959afd1 namespace: kserve-ci-e2e-test resourceVersion: "24241" uid: 0f16fc36-b146-448a-a696-0cb4a3884fa7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6eae8} kind: Pod name: splitter-graph-6eae8-7cdb674958-h2pjc namespace: kserve-ci-e2e-test resourceVersion: "24215" uid: 08be6d3f-7873-46f4-a259-a0e4467eca34 kind: Event lastTimestamp: "2026-03-18T17:27:09Z" message: Stopping container splitter-graph-6eae8 metadata: creationTimestamp: "2026-03-18T17:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:27:09Z" name: splitter-graph-6eae8-7cdb674958-h2pjc.189dff91c8cf94a5 namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: aa91bec2-b6ec-4d26-85b4-442a6347d72f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6eae8} kind: Pod name: splitter-graph-6eae8-7cdb674958-h2pjc namespace: kserve-ci-e2e-test resourceVersion: "24215" uid: 08be6d3f-7873-46f4-a259-a0e4467eca34 kind: Event lastTimestamp: "2026-03-18T17:27:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:27:37Z" name: splitter-graph-6eae8-7cdb674958-h2pjc.189dff925b030b8e namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: a05a99b8-c50f-4b0e-950e-43ba8bc23a20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-6eae8-7cdb674958 namespace: kserve-ci-e2e-test resourceVersion: "24210" uid: f4f87b80-a086-4306-9a96-924af36cd91e kind: Event lastTimestamp: "2026-03-18T17:18:55Z" message: 'Created pod: splitter-graph-6eae8-7cdb674958-h2pjc' metadata: creationTimestamp: "2026-03-18T17:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:55Z" name: splitter-graph-6eae8-7cdb674958.189dff1e96c59ecf namespace: kserve-ci-e2e-test resourceVersion: "24214" uid: 8f3385c4-d2da-430a-bd3b-0626f0934657 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "24208" uid: e8f24961-4b07-469d-8846-c8210358ce25 kind: Event lastTimestamp: "2026-03-18T17:18:55Z" message: Scaled up replica set splitter-graph-6eae8-7cdb674958 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:55Z" name: splitter-graph-6eae8.189dff1e961a5d70 namespace: kserve-ci-e2e-test resourceVersion: "24211" uid: c76d8489-d922-40c7-b6af-a122ba56b936 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "24225" uid: 5eee180e-1082-41bb-a23a-94d3cf35ba21 kind: Event lastTimestamp: "2026-03-18T17:18:55Z" message: InferenceGraph [splitter-graph-6eae8] is Ready metadata: creationTimestamp: "2026-03-18T17:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:18:55Z" name: splitter-graph-6eae8.189dff1e99a6642f namespace: kserve-ci-e2e-test resourceVersion: "24227" uid: 3a48dce6-4153-4c28-bd89-13d0540b6991 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:19:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "24209" uid: 1eaf09bb-cacd-493c-b3b1-d7808439b920 kind: Event lastTimestamp: "2026-03-18T17:19:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:19:10Z" name: splitter-graph-6eae8.189dff22147f7510 namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 04fb1562-4280-463e-88ff-f479f316408c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:19:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "24209" uid: 1eaf09bb-cacd-493c-b3b1-d7808439b920 kind: Event lastTimestamp: "2026-03-18T17:19:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:19:10Z" name: splitter-graph-6eae8.189dff2214ddcf67 namespace: kserve-ci-e2e-test resourceVersion: "24312" uid: 9bb2223b-e00f-4974-896b-3f31692af5a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:19:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 1eaf09bb-cacd-493c-b3b1-d7808439b920 kind: Event lastTimestamp: "2026-03-18T17:19:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:19:25Z" name: splitter-graph-6eae8.189dff2593784a39 namespace: kserve-ci-e2e-test resourceVersion: "24383" uid: 38034086-369b-45ad-9815-aa9cf309d1b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:19:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 1eaf09bb-cacd-493c-b3b1-d7808439b920 kind: Event lastTimestamp: "2026-03-18T17:19:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:19:25Z" name: splitter-graph-6eae8.189dff2593d49af9 namespace: kserve-ci-e2e-test resourceVersion: "24385" uid: e28756cf-799b-4005-8492-6a035d882e83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:16:04.770535Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 317736d7-450d-41e3-b7bf-0cd269b30334 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:16:04Z" name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr.189dfef6ee7ed1e1 namespace: kserve-ci-e2e-test resourceVersion: "22572" uid: c9d2496b-acee-4031-9ed4-4f18ba346d54 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr namespace: kserve-ci-e2e-test resourceVersion: "22577" uid: 317736d7-450d-41e3-b7bf-0cd269b30334 kind: Event lastTimestamp: "2026-03-18T17:16:05Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:16:05Z" name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr.189dfef70865bcaa namespace: kserve-ci-e2e-test resourceVersion: "22602" uid: 56ef88bc-2d02-40fa-a5bf-c5fba6935864 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr namespace: kserve-ci-e2e-test resourceVersion: "22566" uid: 317736d7-450d-41e3-b7bf-0cd269b30334 kind: Event lastTimestamp: "2026-03-18T17:16:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:05Z" name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr.189dfef709ab80fd namespace: kserve-ci-e2e-test resourceVersion: "22605" uid: 8412eef8-85e6-4a66-a996-6cdae8c1bf49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr namespace: kserve-ci-e2e-test resourceVersion: "22566" uid: 317736d7-450d-41e3-b7bf-0cd269b30334 kind: Event lastTimestamp: "2026-03-18T17:16:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:05Z" name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr.189dfef71099b23b namespace: kserve-ci-e2e-test resourceVersion: "22613" uid: 4eee79f5-74cd-4faa-bbc6-22b685148d0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr namespace: kserve-ci-e2e-test resourceVersion: "22566" uid: 317736d7-450d-41e3-b7bf-0cd269b30334 kind: Event lastTimestamp: "2026-03-18T17:16:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:05Z" name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr.189dfef71116a688 namespace: kserve-ci-e2e-test resourceVersion: "22614" uid: e1ae3d93-718e-4ea1-b7ec-692c7d148dd4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T17:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr namespace: kserve-ci-e2e-test resourceVersion: "22566" uid: 317736d7-450d-41e3-b7bf-0cd269b30334 kind: Event lastTimestamp: "2026-03-18T17:17:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:36Z" name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr.189dfef7284c1a52 namespace: kserve-ci-e2e-test resourceVersion: "23710" uid: 4d90c3ba-840b-4730-8b12-526ac54f97bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr namespace: kserve-ci-e2e-test resourceVersion: "22566" uid: 317736d7-450d-41e3-b7bf-0cd269b30334 kind: Event lastTimestamp: "2026-03-18T17:17:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:34Z" name: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr.189dff0beea7e45e namespace: kserve-ci-e2e-test resourceVersion: "23608" uid: 1bd3c844-4f8c-4254-b8b6-7700dbe67344 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0c4f8-predictor-67fb899dc9 namespace: kserve-ci-e2e-test resourceVersion: "22527" uid: 93f4015f-98d3-4844-88c8-e225d9ad31af kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: 'Created pod: success-200-isvc-0c4f8-predictor-67fb899dc9-88dnr' metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:04Z" name: success-200-isvc-0c4f8-predictor-67fb899dc9.189dfef6edfa1036 namespace: kserve-ci-e2e-test resourceVersion: "22565" uid: 4bc20bb8-b8fe-4c24-9606-df5472ff82ca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22524" uid: 911e9987-c5da-436f-912c-8f3901b850da kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: Scaled up replica set success-200-isvc-0c4f8-predictor-67fb899dc9 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:04Z" name: success-200-isvc-0c4f8-predictor.189dfef6e6d89bb2 namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: b7a8235c-bf3b-4eda-9c38-41f8d5124251 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:16:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22526" uid: 038b481e-f8a3-4903-9bd2-db7bf6fd7698 kind: Event lastTimestamp: "2026-03-18T17:16:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:16:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:49Z" name: success-200-isvc-0c4f8-predictor.189dfefa65375339 namespace: kserve-ci-e2e-test resourceVersion: "22984" uid: e4b49604-9a63-443c-987b-a009d308cad9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:16:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22526" uid: 038b481e-f8a3-4903-9bd2-db7bf6fd7698 kind: Event lastTimestamp: "2026-03-18T17:16:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:16:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:49Z" name: success-200-isvc-0c4f8-predictor.189dfefa657ce1cc namespace: kserve-ci-e2e-test resourceVersion: "22985" uid: 2ae214d8-9bf8-4b8f-8871-662ec9987a77 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:17:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22703" uid: 038b481e-f8a3-4903-9bd2-db7bf6fd7698 kind: Event lastTimestamp: "2026-03-18T17:17:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:34Z" name: success-200-isvc-0c4f8-predictor.189dff04e23e3fb7 namespace: kserve-ci-e2e-test resourceVersion: "23588" uid: 87d50e56-6a8c-4144-bc95-cc6a7bf34c2e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T17:17:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0c4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22703" uid: 038b481e-f8a3-4903-9bd2-db7bf6fd7698 kind: Event lastTimestamp: "2026-03-18T17:17:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:34Z" name: success-200-isvc-0c4f8-predictor.189dff04e2810637 namespace: kserve-ci-e2e-test resourceVersion: "23589" uid: 1c31db6f-cd1c-4898-9f44-51ff45356c23 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: 1a9c4ae8-c95b-4914-9c4b-7332ac15723b kind: Event lastTimestamp: "2026-03-18T17:17:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:06Z" name: success-200-isvc-0c4f8.189dfef6e6a4ac72 namespace: kserve-ci-e2e-test resourceVersion: "23093" uid: 314a2c8d-967a-4bc2-a127-4afb23392c35 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: 1a9c4ae8-c95b-4914-9c4b-7332ac15723b kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-0c4f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0c4f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:16:04Z" name: success-200-isvc-0c4f8.189dfef6e70f0ab2 namespace: kserve-ci-e2e-test resourceVersion: "22534" uid: 47d041a3-1f4b-4574-b53b-2578d5ff594a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: 1a9c4ae8-c95b-4914-9c4b-7332ac15723b kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0c4f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:16:04Z" name: success-200-isvc-0c4f8.189dfef6e70f54f8 namespace: kserve-ci-e2e-test resourceVersion: "22537" uid: cc5d3a45-688d-4d0f-a650-c0477680c1c2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0c4f8 namespace: kserve-ci-e2e-test resourceVersion: "23091" uid: 1a9c4ae8-c95b-4914-9c4b-7332ac15723b kind: Event lastTimestamp: "2026-03-18T17:17:06Z" message: InferenceService [success-200-isvc-0c4f8] is Ready metadata: creationTimestamp: "2026-03-18T17:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:06Z" name: success-200-isvc-0c4f8.189dff056006b4fd namespace: kserve-ci-e2e-test resourceVersion: "23092" uid: 490bf452-b0c9-4eba-b3ef-663e57c2bf38 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:05:38.980564Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-18dc8-predictor-8c499d9c-282kw namespace: kserve-ci-e2e-test resourceVersion: "19035" uid: 0ec39ecc-1be3-42be-86e0-f176bf8e08e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-18dc8-predictor-8c499d9c-282kw to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:05:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:05:38Z" name: success-200-isvc-18dc8-predictor-8c499d9c-282kw.189dfe653a81ac23 namespace: kserve-ci-e2e-test resourceVersion: "19040" uid: f08240a1-2447-48d6-9126-3d7576e98ad0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-18dc8-predictor-8c499d9c-282kw namespace: kserve-ci-e2e-test resourceVersion: "19045" uid: 0ec39ecc-1be3-42be-86e0-f176bf8e08e1 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:05:39Z" name: success-200-isvc-18dc8-predictor-8c499d9c-282kw.189dfe6553ea5a31 namespace: kserve-ci-e2e-test resourceVersion: "19092" uid: abe45d18-4571-4524-b6e9-d550aca6fda8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-18dc8-predictor-8c499d9c-282kw namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: 0ec39ecc-1be3-42be-86e0-f176bf8e08e1 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:39Z" name: success-200-isvc-18dc8-predictor-8c499d9c-282kw.189dfe65554253f1 namespace: kserve-ci-e2e-test resourceVersion: "19094" uid: d52d60b7-65e3-4a52-af59-4899c86138b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-18dc8-predictor-8c499d9c-282kw namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: 0ec39ecc-1be3-42be-86e0-f176bf8e08e1 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:39Z" name: success-200-isvc-18dc8-predictor-8c499d9c-282kw.189dfe655c0ddeda namespace: kserve-ci-e2e-test resourceVersion: "19097" uid: 51fee020-e5f6-413c-a350-33133c44e9b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-18dc8-predictor-8c499d9c-282kw namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: 0ec39ecc-1be3-42be-86e0-f176bf8e08e1 kind: Event lastTimestamp: "2026-03-18T17:05:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:39Z" name: success-200-isvc-18dc8-predictor-8c499d9c-282kw.189dfe655c8259d8 namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: f5295d3f-ebe4-49e1-853a-ff29f46a59f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-18dc8-predictor-8c499d9c-282kw namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: 0ec39ecc-1be3-42be-86e0-f176bf8e08e1 kind: Event lastTimestamp: "2026-03-18T17:07:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:10Z" name: success-200-isvc-18dc8-predictor-8c499d9c-282kw.189dfe65651ab891 namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: 6cda5c62-dabd-484b-a33e-cd39fb6c1bfa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-18dc8-predictor-8c499d9c-282kw namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: 0ec39ecc-1be3-42be-86e0-f176bf8e08e1 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-18dc8-predictor-8c499d9c-282kw.189dfe7a3fd72009 namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 2e465ac3-eb72-42ae-b216-813510a0dcd2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-18dc8-predictor-8c499d9c namespace: kserve-ci-e2e-test resourceVersion: "19017" uid: 175b6f33-9bfd-4e93-9eb9-8c16aa481a8e kind: Event lastTimestamp: "2026-03-18T17:05:38Z" message: 'Created pod: success-200-isvc-18dc8-predictor-8c499d9c-282kw' metadata: creationTimestamp: "2026-03-18T17:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:05:38Z" name: success-200-isvc-18dc8-predictor-8c499d9c.189dfe653a292abf namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: eb82efd4-3006-4bad-90b7-5c0e6cf67a36 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-18dc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19014" uid: 01de6f40-ebad-4439-99f2-07e2156687fe kind: Event lastTimestamp: "2026-03-18T17:05:38Z" message: Scaled up replica set success-200-isvc-18dc8-predictor-8c499d9c from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:05:38Z" name: success-200-isvc-18dc8-predictor.189dfe65383aad80 namespace: kserve-ci-e2e-test resourceVersion: "19023" uid: e01df07e-d371-42a7-9e4b-28d364fbb3df reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:05:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-18dc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19016" uid: e67f990d-bf8b-4ae4-b1a6-e3e21c9d4ae3 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-18dc8-predictor.189dfe68b6be86ce namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 6d1ecd31-a761-4ced-a78a-a29437248e24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:05:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-18dc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19016" uid: e67f990d-bf8b-4ae4-b1a6-e3e21c9d4ae3 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-18dc8-predictor.189dfe68b721cfff namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 4893b71b-8f60-4f37-9bc8-29456a2b1ba3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:05:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-18dc8 namespace: kserve-ci-e2e-test resourceVersion: "19006" uid: f609259d-21d8-4c17-a260-17c32b2d1382 kind: Event lastTimestamp: "2026-03-18T17:06:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:40Z" name: success-200-isvc-18dc8.189dfe65381140c0 namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: 5be845c8-73ce-49fd-b09a-015954f6f41a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-18dc8 namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: f609259d-21d8-4c17-a260-17c32b2d1382 kind: Event lastTimestamp: "2026-03-18T17:06:40Z" message: InferenceService [success-200-isvc-18dc8] is Ready metadata: creationTimestamp: "2026-03-18T17:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:40Z" name: success-200-isvc-18dc8.189dfe739e1a21ac namespace: kserve-ci-e2e-test resourceVersion: "19642" uid: 7b1d8b40-8879-40b1-bfbb-0f7617c26ec6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:06:29.573839Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns namespace: kserve-ci-e2e-test resourceVersion: "19496" uid: 97d9ff2f-d076-4548-83cd-129f594d7745 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-19f83-predictor-b5c694bf5-xkjns to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:06:29Z" name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns.189dfe710219cbbf namespace: kserve-ci-e2e-test resourceVersion: "19500" uid: d111dcdd-f5fa-4699-81b4-3d6f4987284f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns namespace: kserve-ci-e2e-test resourceVersion: "19504" uid: 97d9ff2f-d076-4548-83cd-129f594d7745 kind: Event lastTimestamp: "2026-03-18T17:06:30Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:06:30Z" name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns.189dfe711bca43dc namespace: kserve-ci-e2e-test resourceVersion: "19553" uid: 35d5bb5f-8dd8-4b2c-abfd-30d85c703d3e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: 97d9ff2f-d076-4548-83cd-129f594d7745 kind: Event lastTimestamp: "2026-03-18T17:06:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:30Z" name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns.189dfe711d20aaa5 namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: 05a2bb6b-d547-423b-beda-1841a09a803b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: 97d9ff2f-d076-4548-83cd-129f594d7745 kind: Event lastTimestamp: "2026-03-18T17:06:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:30Z" name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns.189dfe712445369d namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: 072bc221-927a-4d4a-9876-362b7c7612f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: 97d9ff2f-d076-4548-83cd-129f594d7745 kind: Event lastTimestamp: "2026-03-18T17:06:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:30Z" name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns.189dfe7124c34e61 namespace: kserve-ci-e2e-test resourceVersion: "19559" uid: 090f9388-1c6a-4cd7-b98e-a5fcc87ff7a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T17:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: 97d9ff2f-d076-4548-83cd-129f594d7745 kind: Event lastTimestamp: "2026-03-18T17:07:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:21Z" name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns.189dfe714f20fdc0 namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 825098a2-edee-41f8-aba2-b61d68014098 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: 97d9ff2f-d076-4548-83cd-129f594d7745 kind: Event lastTimestamp: "2026-03-18T17:16:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:04Z" name: success-200-isvc-19f83-predictor-b5c694bf5-xkjns.189dfef6e8ecfacf namespace: kserve-ci-e2e-test resourceVersion: "22545" uid: 52fe4536-d7b8-4826-bac9-4cef158e79f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-19f83-predictor-b5c694bf5 namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: c3d38979-4576-4de8-9cb2-c0ae937dbca7 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'Created pod: success-200-isvc-19f83-predictor-b5c694bf5-xkjns' metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:29Z" name: success-200-isvc-19f83-predictor-b5c694bf5.189dfe7101b5a150 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: e2331126-a382-44f8-b15f-c79238381356 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 5865a79b-911e-40a8-bd52-9f8273430093 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: Scaled up replica set success-200-isvc-19f83-predictor-b5c694bf5 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:29Z" name: success-200-isvc-19f83-predictor.189dfe71006186bf namespace: kserve-ci-e2e-test resourceVersion: "19491" uid: 9e25a5d4-564b-4e76-b173-014b3fc9c4a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:06:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19488" uid: 2e062109-5c9b-40f4-a0ab-6dc70718ee43 kind: Event lastTimestamp: "2026-03-18T17:06:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:59Z" name: success-200-isvc-19f83-predictor.189dfe747ee0bd21 namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: e2ec1e7b-1361-4a51-9eab-9e3bf6874b67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:06:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19488" uid: 2e062109-5c9b-40f4-a0ab-6dc70718ee43 kind: Event lastTimestamp: "2026-03-18T17:06:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:59Z" name: success-200-isvc-19f83-predictor.189dfe747f463a4b namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 0c2459d3-a7be-4096-9c05-19f3930b5b0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:07:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: 2e062109-5c9b-40f4-a0ab-6dc70718ee43 kind: Event lastTimestamp: "2026-03-18T17:07:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:59Z" name: success-200-isvc-19f83-predictor.189dfe7b7ca782a7 namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 08010c1d-820c-4d75-9939-84eb6f3f9235 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:07:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-19f83-predictor namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: 2e062109-5c9b-40f4-a0ab-6dc70718ee43 kind: Event lastTimestamp: "2026-03-18T17:07:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:59Z" name: success-200-isvc-19f83-predictor.189dfe7b7ce558d1 namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: bc8af119-3e57-426a-ba18-5d935f2d2131 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-19f83 namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: 6912406b-12d3-4346-a4be-5c1e87230232 kind: Event lastTimestamp: "2026-03-18T17:07:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:07:31Z" name: success-200-isvc-19f83.189dfe710039dae3 namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: c756a63e-1bc8-4813-bcd0-141374f36bcc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-19f83 namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: 6912406b-12d3-4346-a4be-5c1e87230232 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-19f83": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-19f83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:29Z" name: success-200-isvc-19f83.189dfe710088d05f namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: 8842da0e-6440-496f-9016-5cc6e4c556ab reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-19f83 namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: 6912406b-12d3-4346-a4be-5c1e87230232 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-19f83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:29Z" name: success-200-isvc-19f83.189dfe7100891ae1 namespace: kserve-ci-e2e-test resourceVersion: "19494" uid: 0e7fc11a-85f2-4af4-b54b-4506ce2af73c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-19f83 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 6912406b-12d3-4346-a4be-5c1e87230232 kind: Event lastTimestamp: "2026-03-18T17:07:31Z" message: InferenceService [success-200-isvc-19f83] is Ready metadata: creationTimestamp: "2026-03-18T17:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:07:31Z" name: success-200-isvc-19f83.189dfe7f86d5de05 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 575cfadc-b72b-4f6b-be02-9ea07fa77d1b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:04:59.269203Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq namespace: kserve-ci-e2e-test resourceVersion: "18709" uid: 4ee55d1a-665a-4b72-b65e-2e82d8f0f04c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4ddda-predictor-5c87999b7-2hjjq to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq.189dfe5bfb866a07 namespace: kserve-ci-e2e-test resourceVersion: "18718" uid: b9679539-2331-41a4-8e46-2ad40268ec45 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq namespace: kserve-ci-e2e-test resourceVersion: "18722" uid: 4ee55d1a-665a-4b72-b65e-2e82d8f0f04c kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq.189dfe5c14af7c8c namespace: kserve-ci-e2e-test resourceVersion: "18768" uid: d70bfad3-9f46-4cdd-a705-4ef6f2fac4c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 4ee55d1a-665a-4b72-b65e-2e82d8f0f04c kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq.189dfe5c161ebdea namespace: kserve-ci-e2e-test resourceVersion: "18770" uid: 9be07d52-4450-4c45-9eb4-7338b4180818 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 4ee55d1a-665a-4b72-b65e-2e82d8f0f04c kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq.189dfe5c1dabd043 namespace: kserve-ci-e2e-test resourceVersion: "18774" uid: ecc43163-8784-4cee-911a-d58078ab1220 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 4ee55d1a-665a-4b72-b65e-2e82d8f0f04c kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq.189dfe5c1e22d92e namespace: kserve-ci-e2e-test resourceVersion: "18775" uid: d241180a-4702-4fb9-aa66-f02846b9b911 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 4ee55d1a-665a-4b72-b65e-2e82d8f0f04c kind: Event lastTimestamp: "2026-03-18T17:06:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:31Z" name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq.189dfe5c485422f4 namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 70675242-52fc-4f12-88fb-34c688f09be3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 4ee55d1a-665a-4b72-b65e-2e82d8f0f04c kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:06:29Z" name: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq.189dfe7107e4557b namespace: kserve-ci-e2e-test resourceVersion: "19524" uid: 457c0bbd-1ec8-462e-964f-d4b185e09266 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4ddda-predictor-5c87999b7 namespace: kserve-ci-e2e-test resourceVersion: "18696" uid: 4b37c32c-5ed9-4811-8f08-74a191d76334 kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: 'Created pod: success-200-isvc-4ddda-predictor-5c87999b7-2hjjq' metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-4ddda-predictor-5c87999b7.189dfe5bfb0a4fc0 namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: 1ab7e51d-cc54-4f24-a992-065bf130e535 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18694" uid: b58bcade-d895-4377-8402-0141bcd17120 kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: Scaled up replica set success-200-isvc-4ddda-predictor-5c87999b7 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-4ddda-predictor.189dfe5bf642d4ee namespace: kserve-ci-e2e-test resourceVersion: "18698" uid: e7332cde-6148-4cc4-a39b-3d7f65a8ba18 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18695" uid: db1887c8-5f2f-4b34-bb40-a6aac1ba63c8 kind: Event lastTimestamp: "2026-03-18T17:05:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:05:14Z" name: success-200-isvc-4ddda-predictor.189dfe5f74c3dcb4 namespace: kserve-ci-e2e-test resourceVersion: "18862" uid: f64116c4-49a4-4129-9127-88c8b8047d06 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18695" uid: db1887c8-5f2f-4b34-bb40-a6aac1ba63c8 kind: Event lastTimestamp: "2026-03-18T17:05:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:05:14Z" name: success-200-isvc-4ddda-predictor.189dfe5f7524b568 namespace: kserve-ci-e2e-test resourceVersion: "18864" uid: 8342512f-5a4c-4a84-a0a4-a596baa857da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: db1887c8-5f2f-4b34-bb40-a6aac1ba63c8 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:29Z" name: success-200-isvc-4ddda-predictor.189dfe62f3d28906 namespace: kserve-ci-e2e-test resourceVersion: "19458" uid: 662eb0ee-06b2-42cc-8f26-d4504d20b2ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4ddda-predictor namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: db1887c8-5f2f-4b34-bb40-a6aac1ba63c8 kind: Event lastTimestamp: "2026-03-18T17:06:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:06:29Z" name: success-200-isvc-4ddda-predictor.189dfe62f41d2d27 namespace: kserve-ci-e2e-test resourceVersion: "19459" uid: 6bfab158-e113-476a-b9f9-4537200dc74c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4ddda namespace: kserve-ci-e2e-test resourceVersion: "18684" uid: e1e829c3-bd48-4465-ab1f-40349d3ff724 kind: Event lastTimestamp: "2026-03-18T17:06:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:01Z" name: success-200-isvc-4ddda.189dfe5bf61d63bb namespace: kserve-ci-e2e-test resourceVersion: "19249" uid: e3c9ba08-c0a3-4ee0-b246-f0811c0b6b9e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4ddda namespace: kserve-ci-e2e-test resourceVersion: "18684" uid: e1e829c3-bd48-4465-ab1f-40349d3ff724 kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: 'Failed to update status for InferenceService "success-200-isvc-4ddda": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4ddda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-4ddda.189dfe5bf6840b72 namespace: kserve-ci-e2e-test resourceVersion: "18700" uid: 2f10924c-47f9-4ba5-997e-40b2458a8be8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4ddda namespace: kserve-ci-e2e-test resourceVersion: "18684" uid: e1e829c3-bd48-4465-ab1f-40349d3ff724 kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4ddda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-4ddda.189dfe5bf6843997 namespace: kserve-ci-e2e-test resourceVersion: "18702" uid: 40e520e7-0bf4-4812-94bd-b0b6736b0653 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:06:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4ddda namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: e1e829c3-bd48-4465-ab1f-40349d3ff724 kind: Event lastTimestamp: "2026-03-18T17:06:01Z" message: InferenceService [success-200-isvc-4ddda] is Ready metadata: creationTimestamp: "2026-03-18T17:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:06:01Z" name: success-200-isvc-4ddda.189dfe6a803a6cf8 namespace: kserve-ci-e2e-test resourceVersion: "19248" uid: 4aeaaf26-659e-4c9c-b532-6a906d208a8b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:17:35.083826Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6eae8-predictor-7b94df567-r9c56 namespace: kserve-ci-e2e-test resourceVersion: "23625" uid: 98b3763c-585d-418a-a124-a44e45ae1196 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6eae8-predictor-7b94df567-r9c56 to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:17:35Z" name: success-200-isvc-6eae8-predictor-7b94df567-r9c56.189dff0bf59641f2 namespace: kserve-ci-e2e-test resourceVersion: "23629" uid: 7f1a67f5-85b7-4da6-827f-fbbe7bf53dc9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6eae8-predictor-7b94df567-r9c56 namespace: kserve-ci-e2e-test resourceVersion: "23633" uid: 98b3763c-585d-418a-a124-a44e45ae1196 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: Add eth0 [10.133.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:17:35Z" name: success-200-isvc-6eae8-predictor-7b94df567-r9c56.189dff0c0ecedc1f namespace: kserve-ci-e2e-test resourceVersion: "23683" uid: b00c3bcf-386d-4f4a-9a18-75bb3b6b42dd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6eae8-predictor-7b94df567-r9c56 namespace: kserve-ci-e2e-test resourceVersion: "23627" uid: 98b3763c-585d-418a-a124-a44e45ae1196 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:35Z" name: success-200-isvc-6eae8-predictor-7b94df567-r9c56.189dff0c0ffe210e namespace: kserve-ci-e2e-test resourceVersion: "23685" uid: d68857ac-fe2a-480b-9202-3092412a3e17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6eae8-predictor-7b94df567-r9c56 namespace: kserve-ci-e2e-test resourceVersion: "23627" uid: 98b3763c-585d-418a-a124-a44e45ae1196 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:35Z" name: success-200-isvc-6eae8-predictor-7b94df567-r9c56.189dff0c16e9ee1e namespace: kserve-ci-e2e-test resourceVersion: "23689" uid: f0ca91ff-b912-4768-b779-24b509b924b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6eae8-predictor-7b94df567-r9c56 namespace: kserve-ci-e2e-test resourceVersion: "23627" uid: 98b3763c-585d-418a-a124-a44e45ae1196 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:17:35Z" name: success-200-isvc-6eae8-predictor-7b94df567-r9c56.189dff0c17612bb8 namespace: kserve-ci-e2e-test resourceVersion: "23690" uid: e9df7166-ab10-4cd6-8923-dc6472f1cd50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T17:17:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6eae8-predictor-7b94df567-r9c56 namespace: kserve-ci-e2e-test resourceVersion: "23627" uid: 98b3763c-585d-418a-a124-a44e45ae1196 kind: Event lastTimestamp: "2026-03-18T17:18:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:27Z" name: success-200-isvc-6eae8-predictor-7b94df567-r9c56.189dff0c2e8b2c74 namespace: kserve-ci-e2e-test resourceVersion: "24046" uid: d1cb4e49-db86-416c-b966-22a1beb05b82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6eae8-predictor-7b94df567-r9c56 namespace: kserve-ci-e2e-test resourceVersion: "23627" uid: 98b3763c-585d-418a-a124-a44e45ae1196 kind: Event lastTimestamp: "2026-03-18T17:27:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:27:10Z" name: success-200-isvc-6eae8-predictor-7b94df567-r9c56.189dff91d3282689 namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: eb41d4f7-a510-49b9-a8d9-1f8b5518d79d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6eae8-predictor-7b94df567 namespace: kserve-ci-e2e-test resourceVersion: "23618" uid: ff2d0268-6bdd-4d4c-8dbc-9e97fdd12959 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: 'Created pod: success-200-isvc-6eae8-predictor-7b94df567-r9c56' metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:35Z" name: success-200-isvc-6eae8-predictor-7b94df567.189dff0bf51aa6ac namespace: kserve-ci-e2e-test resourceVersion: "23626" uid: dbe9ec8b-7bb7-479d-a51d-7b59b349397a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23616" uid: e917a65f-e76a-43fd-9bde-89a3540bd11b kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: Scaled up replica set success-200-isvc-6eae8-predictor-7b94df567 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:35Z" name: success-200-isvc-6eae8-predictor.189dff0bf3d59c8f namespace: kserve-ci-e2e-test resourceVersion: "23619" uid: 5a0c2a56-6463-4cd7-9695-20af583fc1f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23617" uid: b95f8344-30df-46b1-88c3-7041020ef3ea kind: Event lastTimestamp: "2026-03-18T17:17:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:50Z" name: success-200-isvc-6eae8-predictor.189dff0f7272e9dd namespace: kserve-ci-e2e-test resourceVersion: "23805" uid: 1ec6bc88-009f-459b-925f-241ec8745ce1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23617" uid: b95f8344-30df-46b1-88c3-7041020ef3ea kind: Event lastTimestamp: "2026-03-18T17:17:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:50Z" name: success-200-isvc-6eae8-predictor.189dff0f72d1e7a8 namespace: kserve-ci-e2e-test resourceVersion: "23807" uid: 2a8da80e-8b9c-431c-b373-bfff406f8d4c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:18:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23806" uid: b95f8344-30df-46b1-88c3-7041020ef3ea kind: Event lastTimestamp: "2026-03-18T17:19:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:19:05Z" name: success-200-isvc-6eae8-predictor.189dff12f1ed3644 namespace: kserve-ci-e2e-test resourceVersion: "24286" uid: e1865ba5-4d6a-4e91-9c09-2aabe4f1728d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:18:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6eae8-predictor namespace: kserve-ci-e2e-test resourceVersion: "23806" uid: b95f8344-30df-46b1-88c3-7041020ef3ea kind: Event lastTimestamp: "2026-03-18T17:19:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:19:05Z" name: success-200-isvc-6eae8-predictor.189dff12f24ea247 namespace: kserve-ci-e2e-test resourceVersion: "24287" uid: fc999de6-0387-48ed-8497-406eef43d234 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "23595" uid: 583c44b1-3079-43c3-8960-67cf08f9a963 kind: Event lastTimestamp: "2026-03-18T17:18:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:18:37Z" name: success-200-isvc-6eae8.189dff0bf3d11bd9 namespace: kserve-ci-e2e-test resourceVersion: "24113" uid: 98de0409-e27b-4ac7-bf27-1d3f27196a24 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "23595" uid: 583c44b1-3079-43c3-8960-67cf08f9a963 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: 'Failed to update status for InferenceService "success-200-isvc-6eae8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6eae8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:35Z" name: success-200-isvc-6eae8.189dff0bf45df3c5 namespace: kserve-ci-e2e-test resourceVersion: "23622" uid: 115c3b36-819e-4199-a3e4-49c3f4342330 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "23595" uid: 583c44b1-3079-43c3-8960-67cf08f9a963 kind: Event lastTimestamp: "2026-03-18T17:17:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6eae8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:35Z" name: success-200-isvc-6eae8.189dff0bf45e3c74 namespace: kserve-ci-e2e-test resourceVersion: "23624" uid: 38eb6605-90cf-4e42-8ade-cdaf849112dd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6eae8 namespace: kserve-ci-e2e-test resourceVersion: "24109" uid: 583c44b1-3079-43c3-8960-67cf08f9a963 kind: Event lastTimestamp: "2026-03-18T17:18:37Z" message: InferenceService [success-200-isvc-6eae8] is Ready metadata: creationTimestamp: "2026-03-18T17:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:18:37Z" name: success-200-isvc-6eae8.189dff1a69a53eb3 namespace: kserve-ci-e2e-test resourceVersion: "24112" uid: 29fde7d1-f3fb-46ba-9b30-0030505e2fc1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:07:09.272362Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv namespace: kserve-ci-e2e-test resourceVersion: "19896" uid: bb38c4ad-38c5-4006-9eab-4a67b9575f24 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv.189dfe7a40512984 namespace: kserve-ci-e2e-test resourceVersion: "19901" uid: 501c0104-79a0-4257-871d-12f2dd1b95ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: bb38c4ad-38c5-4006-9eab-4a67b9575f24 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv.189dfe7a5ab5fece namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: 20ac3c75-d130-4726-ba65-7f44c6cb045f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: bb38c4ad-38c5-4006-9eab-4a67b9575f24 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv.189dfe7a5bfdcb0f namespace: kserve-ci-e2e-test resourceVersion: "19955" uid: 65175dab-4409-4a6d-a63f-21170c5fd3cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: bb38c4ad-38c5-4006-9eab-4a67b9575f24 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv.189dfe7a62627420 namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: a4e1b09f-b8ad-4448-b788-62b137f061bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: bb38c4ad-38c5-4006-9eab-4a67b9575f24 kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv.189dfe7a62da73b3 namespace: kserve-ci-e2e-test resourceVersion: "19961" uid: 43ab51bf-d78b-41ec-8a2a-da24586b8936 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T17:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: bb38c4ad-38c5-4006-9eab-4a67b9575f24 kind: Event lastTimestamp: "2026-03-18T17:08:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:08:01Z" name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv.189dfe7a6bbae467 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: d4b53977-b577-498d-8cb3-5d5dd0f67d92 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: bb38c4ad-38c5-4006-9eab-4a67b9575f24 kind: Event lastTimestamp: "2026-03-18T17:16:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:16:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:43Z" name: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv.189dff0010394eb0 namespace: kserve-ci-e2e-test resourceVersion: "22857" uid: 3018d6f2-2d39-4754-8829-594e68305a49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8d83a-predictor-7f9fd554b9 namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 7e05d365-2715-4586-8716-f822ddeef68c kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: 'Created pod: success-200-isvc-8d83a-predictor-7f9fd554b9-gb4cv' metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-8d83a-predictor-7f9fd554b9.189dfe7a3fec93bb namespace: kserve-ci-e2e-test resourceVersion: "19898" uid: 84cf1779-f24f-47b2-89fb-1d1119eccadb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8d83a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: cd0ae3bf-ca8f-43cc-a170-caec97bb609d kind: Event lastTimestamp: "2026-03-18T17:07:09Z" message: Scaled up replica set success-200-isvc-8d83a-predictor-7f9fd554b9 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:09Z" name: success-200-isvc-8d83a-predictor.189dfe7a3e77ce64 namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: 1ff508d7-4ba7-46b6-add0-e8b4808349db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d83a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: 81628984-5a88-405c-893d-f09a6b1a61c4 kind: Event lastTimestamp: "2026-03-18T17:07:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:39Z" name: success-200-isvc-8d83a-predictor.189dfe7dbd1be293 namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: fa83a91b-67df-4ad6-85e3-2c8ee8f6b3f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d83a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: 81628984-5a88-405c-893d-f09a6b1a61c4 kind: Event lastTimestamp: "2026-03-18T17:07:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:07:39Z" name: success-200-isvc-8d83a-predictor.189dfe7dbd7b311a namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: 3f819592-f252-47f7-b4bb-a8c33e550f35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:07:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d83a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 81628984-5a88-405c-893d-f09a6b1a61c4 kind: Event lastTimestamp: "2026-03-18T17:08:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:39Z" name: success-200-isvc-8d83a-predictor.189dfe84bb39a3f4 namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: f4a4987d-cf44-4d71-9a56-33d952b9e3c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:07:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d83a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 81628984-5a88-405c-893d-f09a6b1a61c4 kind: Event lastTimestamp: "2026-03-18T17:08:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:08:39Z" name: success-200-isvc-8d83a-predictor.189dfe84bb959ccf namespace: kserve-ci-e2e-test resourceVersion: "20573" uid: 692d0ed4-4a38-4b42-b6ba-234632450499 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:07:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d83a namespace: kserve-ci-e2e-test resourceVersion: "19869" uid: 48c9b39e-8409-46c3-bc70-2c00fc95c97e kind: Event lastTimestamp: "2026-03-18T17:08:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:08:11Z" name: success-200-isvc-8d83a.189dfe7a3e7d3ef8 namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 2f50f36d-f1be-4420-8076-9f42edafe242 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:08:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d83a namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 48c9b39e-8409-46c3-bc70-2c00fc95c97e kind: Event lastTimestamp: "2026-03-18T17:08:11Z" message: InferenceService [success-200-isvc-8d83a] is Ready metadata: creationTimestamp: "2026-03-18T17:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:08:11Z" name: success-200-isvc-8d83a.189dfe88a82d24b2 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: fbea7a17-fb99-45de-88f8-94cb1c56b9c9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:56:04.092648Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: e10de7c8-98d3-46a1-87ee-53839a451906 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9913d-predictor-6b8667cb5c-rct6m to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:56:04Z" name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m.189dfddf60842d27 namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: ff714c06-3bd3-4028-91a3-74aff30cfec7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: e10de7c8-98d3-46a1-87ee-53839a451906 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:56:04Z" name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m.189dfddf7a258ac2 namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: fbad468a-ac03-4503-87eb-5064a296f258 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: e10de7c8-98d3-46a1-87ee-53839a451906 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:04Z" name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m.189dfddf7b79e2f3 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c73ca63-a74c-4320-8e4a-fa687f78f87a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: e10de7c8-98d3-46a1-87ee-53839a451906 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:04Z" name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m.189dfddf816c12d9 namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 084f7f73-85b4-498a-a68d-2c9034750cdb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: e10de7c8-98d3-46a1-87ee-53839a451906 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:04Z" name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m.189dfddf81e39a27 namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: e0e72719-fe91-4103-b09e-74a1205fa6dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: e10de7c8-98d3-46a1-87ee-53839a451906 kind: Event lastTimestamp: "2026-03-18T16:56:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:55Z" name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m.189dfddf8c0a75b9 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 09068b83-f70f-417c-a759-2d6e20667149 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:05:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: e10de7c8-98d3-46a1-87ee-53839a451906 kind: Event lastTimestamp: "2026-03-18T17:05:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:38Z" name: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m.189dfe6539ff49ad namespace: kserve-ci-e2e-test resourceVersion: "19036" uid: 61bfd934-2126-4043-adc7-6eca82011a21 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9913d-predictor-6b8667cb5c namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: 4504f1bc-9119-4719-9270-cbae48a225d1 kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: 'Created pod: success-200-isvc-9913d-predictor-6b8667cb5c-rct6m' metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:04Z" name: success-200-isvc-9913d-predictor-6b8667cb5c.189dfddf6020b8a0 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: b8934367-256c-401a-a67e-bd8d865f6032 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 4901de3d-2ec2-4839-8203-7acd5bcfbe7b kind: Event lastTimestamp: "2026-03-18T16:56:04Z" message: Scaled up replica set success-200-isvc-9913d-predictor-6b8667cb5c from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:04Z" name: success-200-isvc-9913d-predictor.189dfddf5eb34394 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 509463e7-3bdf-41d6-98b2-3e2fa3462be6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: 7844a14e-64b8-470f-9459-c544ffe6d781 kind: Event lastTimestamp: "2026-03-18T16:56:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:49Z" name: success-200-isvc-9913d-predictor.189dfde2dd4bc978 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: f40761c6-f7a9-425f-8b2c-ad94bf6b6445 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: 7844a14e-64b8-470f-9459-c544ffe6d781 kind: Event lastTimestamp: "2026-03-18T16:56:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:49Z" name: success-200-isvc-9913d-predictor.189dfde2ddb7e981 namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 5a91ea7d-c6d9-4368-aede-5752f53acdf4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:57:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 7844a14e-64b8-470f-9459-c544ffe6d781 kind: Event lastTimestamp: "2026-03-18T16:57:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:19Z" name: success-200-isvc-9913d-predictor.189dfded5a6371ae namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: 5a2acb3e-8617-405b-8aeb-e5b54b0d9973 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T16:57:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9913d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 7844a14e-64b8-470f-9459-c544ffe6d781 kind: Event lastTimestamp: "2026-03-18T16:57:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:19Z" name: success-200-isvc-9913d-predictor.189dfded5ab9d0b8 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 533fc1ec-9abe-42de-bfd1-1210b44a0d3c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T16:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9913d namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: bb6848f2-325b-4a30-a52a-ee0f9b142cb9 kind: Event lastTimestamp: "2026-03-18T16:57:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:57:05Z" name: success-200-isvc-9913d.189dfddf5ebc48f2 namespace: kserve-ci-e2e-test resourceVersion: "15884" uid: 9ca32dea-5089-4046-85ce-00756f238254 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9913d namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: bb6848f2-325b-4a30-a52a-ee0f9b142cb9 kind: Event lastTimestamp: "2026-03-18T16:57:05Z" message: InferenceService [success-200-isvc-9913d] is Ready metadata: creationTimestamp: "2026-03-18T16:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:57:05Z" name: success-200-isvc-9913d.189dfdedc3cb66d1 namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 476125b8-5e76-4814-86b8-7e98dde07898 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:53:43.954752Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k namespace: kserve-ci-e2e-test resourceVersion: "13512" uid: 0172b66d-e446-422b-a91f-901dcaad88d2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a225b-predictor-b8745d9cc-cqs9k to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T16:53:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:53:43Z" name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k.189dfdbebfa5a3ce namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: 6520d6d9-280a-4070-9640-feee891c69dc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k namespace: kserve-ci-e2e-test resourceVersion: "13524" uid: 0172b66d-e446-422b-a91f-901dcaad88d2 kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:53:44Z" name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k.189dfdbedb615ed3 namespace: kserve-ci-e2e-test resourceVersion: "13601" uid: 0cbcf826-0cfd-4be7-9494-357d179f9e3c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k namespace: kserve-ci-e2e-test resourceVersion: "13516" uid: 0172b66d-e446-422b-a91f-901dcaad88d2 kind: Event lastTimestamp: "2026-03-18T16:53:44Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" metadata: creationTimestamp: "2026-03-18T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:44Z" name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k.189dfdbedcdc5137 namespace: kserve-ci-e2e-test resourceVersion: "13604" uid: 87c32ed6-08a3-4a9b-b86c-b3ff42648b3a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k namespace: kserve-ci-e2e-test resourceVersion: "13516" uid: 0172b66d-e446-422b-a91f-901dcaad88d2 kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" in 6.218s (6.218s including waiting). Image size: 367586442 bytes.' metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k.189dfdc04f805436 namespace: kserve-ci-e2e-test resourceVersion: "13720" uid: ba3bd056-7176-4c08-9b1e-99244b3c6cb8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k namespace: kserve-ci-e2e-test resourceVersion: "13516" uid: 0172b66d-e446-422b-a91f-901dcaad88d2 kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k.189dfdc0541d0ba7 namespace: kserve-ci-e2e-test resourceVersion: "13726" uid: e5cc42f3-bc78-4c91-998d-2168f1ea6abb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k namespace: kserve-ci-e2e-test resourceVersion: "13516" uid: 0172b66d-e446-422b-a91f-901dcaad88d2 kind: Event lastTimestamp: "2026-03-18T16:53:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T16:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:53:50Z" name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k.189dfdc05496ca00 namespace: kserve-ci-e2e-test resourceVersion: "13727" uid: fdcc6967-12f7-4a8b-bb69-8d68fb96ed71 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T16:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k namespace: kserve-ci-e2e-test resourceVersion: "13516" uid: 0172b66d-e446-422b-a91f-901dcaad88d2 kind: Event lastTimestamp: "2026-03-18T16:54:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T16:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:54:43Z" name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k.189dfdc07832d71d namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 89937c06-86c7-472e-8c8b-4e9a3899e97b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k namespace: kserve-ci-e2e-test resourceVersion: "13516" uid: 0172b66d-e446-422b-a91f-901dcaad88d2 kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:24Z" name: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k.189dfdd60eb3a20b namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: f0b9f1cd-7100-45a8-97a0-9e9a8bc1d1da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a225b-predictor-b8745d9cc namespace: kserve-ci-e2e-test resourceVersion: "13504" uid: e71b048a-2a82-4ab9-bc71-4d92559f2de3 kind: Event lastTimestamp: "2026-03-18T16:53:43Z" message: 'Created pod: success-200-isvc-a225b-predictor-b8745d9cc-cqs9k' metadata: creationTimestamp: "2026-03-18T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:43Z" name: success-200-isvc-a225b-predictor-b8745d9cc.189dfdbebf163e7d namespace: kserve-ci-e2e-test resourceVersion: "13514" uid: dbe430c4-d77d-40a4-8047-e533d72d4611 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:53:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13503" uid: 9acb47b5-4976-4e2b-a9fc-fd12a9a88875 kind: Event lastTimestamp: "2026-03-18T16:53:43Z" message: Scaled up replica set success-200-isvc-a225b-predictor-b8745d9cc from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:53:43Z" name: success-200-isvc-a225b-predictor.189dfdbebb23ac6b namespace: kserve-ci-e2e-test resourceVersion: "13506" uid: f2e178ab-103a-48a9-8505-0d52accf24f0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:53:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13505" uid: 13945b19-b37d-4cf7-9b87-ccd49152e212 kind: Event lastTimestamp: "2026-03-18T16:54:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:54:28Z" name: success-200-isvc-a225b-predictor.189dfdc239f1c802 namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: 55777510-e74d-48cb-9471-129c545f1a8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:53:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13505" uid: 13945b19-b37d-4cf7-9b87-ccd49152e212 kind: Event lastTimestamp: "2026-03-18T16:54:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:53:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:54:28Z" name: success-200-isvc-a225b-predictor.189dfdc23a514288 namespace: kserve-ci-e2e-test resourceVersion: "14129" uid: 51c9e6f9-880c-4367-80cf-bcec3bf20a8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:54:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: 13945b19-b37d-4cf7-9b87-ccd49152e212 kind: Event lastTimestamp: "2026-03-18T16:55:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:13Z" name: success-200-isvc-a225b-predictor.189dfdccb669e857 namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: 8d6267b9-9160-4033-8e0b-8026d37e1101 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-03-18T16:54:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a225b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: 13945b19-b37d-4cf7-9b87-ccd49152e212 kind: Event lastTimestamp: "2026-03-18T16:55:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:13Z" name: success-200-isvc-a225b-predictor.189dfdccb6d4783e namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 1aa9a964-8b95-4dbd-891c-1a2e9f75c27e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T16:53:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a225b namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: 8adb4607-ca8f-40b7-b991-ecbdf7874751 kind: Event lastTimestamp: "2026-03-18T16:54:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:54:53Z" name: success-200-isvc-a225b.189dfdbebb3db6e5 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: f73c1222-e576-4167-a584-e9d93f137388 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:54:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a225b namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 8adb4607-ca8f-40b7-b991-ecbdf7874751 kind: Event lastTimestamp: "2026-03-18T16:54:53Z" message: InferenceService [success-200-isvc-a225b] is Ready metadata: creationTimestamp: "2026-03-18T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:54:53Z" name: success-200-isvc-a225b.189dfdceed33d274 namespace: kserve-ci-e2e-test resourceVersion: "14353" uid: 65905562-92da-443d-8a39-d0bafa8ddb9c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:55:24.065250Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k namespace: kserve-ci-e2e-test resourceVersion: "14693" uid: 95a83498-405c-443e-b367-47e3e745d28c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dfe46-predictor-756f46b767-v7t9k to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:55:24Z" name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k.189dfdd60eb2915d namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 2bf8493b-4937-4bdf-b49e-2ccce13a5f58 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: 95a83498-405c-443e-b367-47e3e745d28c kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:55:24Z" name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k.189dfdd6284d5806 namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: ede30abd-50cf-4a1d-9d93-c16fc28d0ce7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 95a83498-405c-443e-b367-47e3e745d28c kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:24Z" name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k.189dfdd629992404 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 76f6a4d6-84d7-4ac6-a08d-2b69a55b11b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 95a83498-405c-443e-b367-47e3e745d28c kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:24Z" name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k.189dfdd631712384 namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: cff5ad02-5b52-42fb-8c78-b01c9a2937cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 95a83498-405c-443e-b367-47e3e745d28c kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:24Z" name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k.189dfdd631e4c62f namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 0ded3647-9461-4604-934f-d2f94f42133d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 95a83498-405c-443e-b367-47e3e745d28c kind: Event lastTimestamp: "2026-03-18T16:56:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:15Z" name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k.189dfdd633a334dc namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 1947524c-7f55-4e9b-b51d-52156f9a4885 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 95a83498-405c-443e-b367-47e3e745d28c kind: Event lastTimestamp: "2026-03-18T17:04:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:04:59Z" name: success-200-isvc-dfe46-predictor-756f46b767-v7t9k.189dfe5bfc0bc515 namespace: kserve-ci-e2e-test resourceVersion: "18720" uid: 0cc966f9-7a3d-4e69-aec4-4f52d6bbc3f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dfe46-predictor-756f46b767 namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: 1aee64c4-57b4-409b-ae96-c44799838d5e kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: 'Created pod: success-200-isvc-dfe46-predictor-756f46b767-v7t9k' metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:24Z" name: success-200-isvc-dfe46-predictor-756f46b767.189dfdd60e58cae6 namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: c289fb72-d6a4-46c5-89bc-ed960756b5a9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dfe46-predictor namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: 3dab17ec-8715-40e4-8ca5-c3e3bf7be6eb kind: Event lastTimestamp: "2026-03-18T16:55:23Z" message: Scaled up replica set success-200-isvc-dfe46-predictor-756f46b767 from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:23Z" name: success-200-isvc-dfe46-predictor.189dfdd60a21f218 namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: 739ffe46-aa76-40c5-973f-8203488e5f7a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T16:55:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dfe46-predictor namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 27d1895b-9ec3-40c2-9d9b-aa95da27b333 kind: Event lastTimestamp: "2026-03-18T16:56:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:54Z" name: success-200-isvc-dfe46-predictor.189dfdd988f37f8a namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 8db0b92b-1843-45a9-9770-873b78232c9b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T16:55:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dfe46-predictor namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 27d1895b-9ec3-40c2-9d9b-aa95da27b333 kind: Event lastTimestamp: "2026-03-18T16:56:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:54Z" name: success-200-isvc-dfe46-predictor.189dfdd98973792b namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 8296dc96-fbd0-4854-aa81-3d618d8092c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T16:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 7f61c1db-bdb5-4755-93e0-5d9736d399c2 kind: Event lastTimestamp: "2026-03-18T16:56:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:56:25Z" name: success-200-isvc-dfe46.189dfdd609fb892c namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: dd5cb675-e51e-491a-8e96-ee86b6ce3e8c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 7f61c1db-bdb5-4755-93e0-5d9736d399c2 kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: 'Failed to update status for InferenceService "success-200-isvc-dfe46": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dfe46": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:24Z" name: success-200-isvc-dfe46.189dfdd60c5b061b namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: b84dd3db-051f-45d9-9631-9d9bb35bcb0d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 7f61c1db-bdb5-4755-93e0-5d9736d399c2 kind: Event lastTimestamp: "2026-03-18T16:55:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dfe46": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:24Z" name: success-200-isvc-dfe46.189dfdd60c5b3c5b namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: 1ca5f25d-a794-435f-9fdb-d87998e57a02 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 7f61c1db-bdb5-4755-93e0-5d9736d399c2 kind: Event lastTimestamp: "2026-03-18T16:56:25Z" message: InferenceService [success-200-isvc-dfe46] is Ready metadata: creationTimestamp: "2026-03-18T16:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:56:25Z" name: success-200-isvc-dfe46.189dfde47128de53 namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: eb3d28b0-e05c-4095-b40a-05fa08fb947e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:16:44.251212Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eb922-predictor-ff68cf985-w7v89 namespace: kserve-ci-e2e-test resourceVersion: "22885" uid: 8ec60ecd-1545-462c-9db3-184ba0c557d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-eb922-predictor-ff68cf985-w7v89 to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:16:44Z" name: success-200-isvc-eb922-predictor-ff68cf985-w7v89.189dff001fba2973 namespace: kserve-ci-e2e-test resourceVersion: "22891" uid: 673bc9ee-0cc0-4fd8-bbbe-06b4c948f9e7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eb922-predictor-ff68cf985-w7v89 namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: 8ec60ecd-1545-462c-9db3-184ba0c557d0 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: Add eth0 [10.133.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:16:44Z" name: success-200-isvc-eb922-predictor-ff68cf985-w7v89.189dff0039406a25 namespace: kserve-ci-e2e-test resourceVersion: "22933" uid: 7941ae3f-e87b-4a96-9655-b852eecc791c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb922-predictor-ff68cf985-w7v89 namespace: kserve-ci-e2e-test resourceVersion: "22889" uid: 8ec60ecd-1545-462c-9db3-184ba0c557d0 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1241" already present on machine metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:44Z" name: success-200-isvc-eb922-predictor-ff68cf985-w7v89.189dff003a799eec namespace: kserve-ci-e2e-test resourceVersion: "22935" uid: 94332e61-8205-443a-9dd4-5edae0e2297f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb922-predictor-ff68cf985-w7v89 namespace: kserve-ci-e2e-test resourceVersion: "22889" uid: 8ec60ecd-1545-462c-9db3-184ba0c557d0 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:44Z" name: success-200-isvc-eb922-predictor-ff68cf985-w7v89.189dff003d88b512 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: afa5d997-1a51-4cb1-93de-5cfd47823b3f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb922-predictor-ff68cf985-w7v89 namespace: kserve-ci-e2e-test resourceVersion: "22889" uid: 8ec60ecd-1545-462c-9db3-184ba0c557d0 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:16:44Z" name: success-200-isvc-eb922-predictor-ff68cf985-w7v89.189dff003df7e477 namespace: kserve-ci-e2e-test resourceVersion: "22938" uid: 489d483b-bc06-4ec9-81a2-ab73bd20e1e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb922-predictor-ff68cf985-w7v89 namespace: kserve-ci-e2e-test resourceVersion: "22889" uid: 8ec60ecd-1545-462c-9db3-184ba0c557d0 kind: Event lastTimestamp: "2026-03-18T17:34:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:34:25Z" name: success-200-isvc-eb922-predictor-ff68cf985-w7v89.189dff0044986bca namespace: kserve-ci-e2e-test resourceVersion: "28414" uid: 1777398c-bdc3-4388-a3a9-96fb46774701 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:34:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb922-predictor-ff68cf985-w7v89 namespace: kserve-ci-e2e-test resourceVersion: "22889" uid: 8ec60ecd-1545-462c-9db3-184ba0c557d0 kind: Event lastTimestamp: "2026-03-18T17:34:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-03-18T17:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:34:23Z" name: success-200-isvc-eb922-predictor-ff68cf985-w7v89.189dfff6cccfc861 namespace: kserve-ci-e2e-test resourceVersion: "28358" uid: 5b5863de-647e-4808-bb10-766f80ede47a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-eb922-predictor-ff68cf985 namespace: kserve-ci-e2e-test resourceVersion: "22877" uid: ecb887ef-8c05-45c6-a43b-375b89eac3a3 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: 'Created pod: success-200-isvc-eb922-predictor-ff68cf985-w7v89' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:44Z" name: success-200-isvc-eb922-predictor-ff68cf985.189dff001f6c8320 namespace: kserve-ci-e2e-test resourceVersion: "22888" uid: 2168c8ff-817d-49c1-840f-166f81dd2e3e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "22874" uid: 427c2b3b-7554-4348-af89-102df69019ea kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: Scaled up replica set success-200-isvc-eb922-predictor-ff68cf985 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:16:44Z" name: success-200-isvc-eb922-predictor.189dff001ae19310 namespace: kserve-ci-e2e-test resourceVersion: "22879" uid: a658d335-7836-4c91-be8b-4c8e69a62755 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:16:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "22876" uid: eb8034f0-11ce-48b7-9a4a-df98ed31c85c kind: Event lastTimestamp: "2026-03-18T17:17:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:14Z" name: success-200-isvc-eb922-predictor.189dff0399858d6c namespace: kserve-ci-e2e-test resourceVersion: "23296" uid: d059b22d-f010-4f85-b44e-59f31b9f55d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:16:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "22876" uid: eb8034f0-11ce-48b7-9a4a-df98ed31c85c kind: Event lastTimestamp: "2026-03-18T17:17:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:17:14Z" name: success-200-isvc-eb922-predictor.189dff0399c1bf11 namespace: kserve-ci-e2e-test resourceVersion: "23297" uid: aefa30a9-5c60-49c9-8530-c22d5e3d6f17 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:17:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "23037" uid: eb8034f0-11ce-48b7-9a4a-df98ed31c85c kind: Event lastTimestamp: "2026-03-18T17:18:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:17:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:14Z" name: success-200-isvc-eb922-predictor.189dff0a971fdc3a namespace: kserve-ci-e2e-test resourceVersion: "23980" uid: ee4380fb-c32a-4b2a-89e6-eac2ccfa6ab9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-03-18T17:17:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb922-predictor namespace: kserve-ci-e2e-test resourceVersion: "23037" uid: eb8034f0-11ce-48b7-9a4a-df98ed31c85c kind: Event lastTimestamp: "2026-03-18T17:18:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T17:17:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:14Z" name: success-200-isvc-eb922-predictor.189dff0a97812ee3 namespace: kserve-ci-e2e-test resourceVersion: "23981" uid: a52d8575-3fcb-4538-8c5b-b8f072b9ee0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eb922 namespace: kserve-ci-e2e-test resourceVersion: "22845" uid: 31156963-565f-4758-86f0-7054334371c7 kind: Event lastTimestamp: "2026-03-18T17:17:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:45Z" name: success-200-isvc-eb922.189dff001e3f4134 namespace: kserve-ci-e2e-test resourceVersion: "23771" uid: c7286f98-0663-447a-b72c-8dfa85ca6269 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eb922 namespace: kserve-ci-e2e-test resourceVersion: "22845" uid: 31156963-565f-4758-86f0-7054334371c7 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-eb922": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-eb922": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:16:44Z" name: success-200-isvc-eb922.189dff001eb7e1c1 namespace: kserve-ci-e2e-test resourceVersion: "22884" uid: 39e11e1b-4a40-4ecf-993f-58089e0d32fe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:16:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eb922 namespace: kserve-ci-e2e-test resourceVersion: "22845" uid: 31156963-565f-4758-86f0-7054334371c7 kind: Event lastTimestamp: "2026-03-18T17:16:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-eb922": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-03-18T17:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:16:44Z" name: success-200-isvc-eb922.189dff001eb82ae4 namespace: kserve-ci-e2e-test resourceVersion: "22886" uid: c4720afd-24b6-4d4b-b962-785a849c0f83 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:17:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eb922 namespace: kserve-ci-e2e-test resourceVersion: "23768" uid: 31156963-565f-4758-86f0-7054334371c7 kind: Event lastTimestamp: "2026-03-18T17:17:45Z" message: InferenceService [success-200-isvc-eb922] is Ready metadata: creationTimestamp: "2026-03-18T17:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:17:45Z" name: success-200-isvc-eb922.189dff0e7c2131a4 namespace: kserve-ci-e2e-test resourceVersion: "23770" uid: d43cb3c1-2c73-4dfe-b0cc-524ded30a37c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:55:13.656058Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a225b-c4ddc8b65-wjsvf namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: 717f798e-7a76-498a-a553-4bf29eec66b0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-a225b-c4ddc8b65-wjsvf to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T16:55:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:55:13Z" name: switch-graph-a225b-c4ddc8b65-wjsvf.189dfdd3a242f85b namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: 507169cb-112f-4604-9d85-6020c8c15d7d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:14Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a225b-c4ddc8b65-wjsvf namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: 717f798e-7a76-498a-a553-4bf29eec66b0 kind: Event lastTimestamp: "2026-03-18T16:55:14Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:55:14Z" name: switch-graph-a225b-c4ddc8b65-wjsvf.189dfdd3bb78fbc4 namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: bbb20eb7-587c-4abb-bea2-384a5886a33b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a225b} kind: Pod name: switch-graph-a225b-c4ddc8b65-wjsvf namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 717f798e-7a76-498a-a553-4bf29eec66b0 kind: Event lastTimestamp: "2026-03-18T16:55:14Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" metadata: creationTimestamp: "2026-03-18T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:14Z" name: switch-graph-a225b-c4ddc8b65-wjsvf.189dfdd3bce32c32 namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: c00d474b-ee46-4454-9f6a-d7ce89c49e5f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a225b} kind: Pod name: switch-graph-a225b-c4ddc8b65-wjsvf namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 717f798e-7a76-498a-a553-4bf29eec66b0 kind: Event lastTimestamp: "2026-03-18T16:55:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" in 1.481s (1.481s including waiting). Image size: 215812660 bytes.' metadata: creationTimestamp: "2026-03-18T16:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:15Z" name: switch-graph-a225b-c4ddc8b65-wjsvf.189dfdd41530a3c8 namespace: kserve-ci-e2e-test resourceVersion: "14580" uid: 2d9df6a2-2028-453b-9013-550785422af2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a225b} kind: Pod name: switch-graph-a225b-c4ddc8b65-wjsvf namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 717f798e-7a76-498a-a553-4bf29eec66b0 kind: Event lastTimestamp: "2026-03-18T16:55:15Z" message: 'Created container: switch-graph-a225b' metadata: creationTimestamp: "2026-03-18T16:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:15Z" name: switch-graph-a225b-c4ddc8b65-wjsvf.189dfdd417fb706b namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: e8d3d69f-74c4-4b97-99ed-51d27e29fca9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a225b} kind: Pod name: switch-graph-a225b-c4ddc8b65-wjsvf namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 717f798e-7a76-498a-a553-4bf29eec66b0 kind: Event lastTimestamp: "2026-03-18T16:55:15Z" message: Started container switch-graph-a225b metadata: creationTimestamp: "2026-03-18T16:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:15Z" name: switch-graph-a225b-c4ddc8b65-wjsvf.189dfdd41872bc81 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 03eac561-61d1-41e1-9e46-67f1e302215d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a225b} kind: Pod name: switch-graph-a225b-c4ddc8b65-wjsvf namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 717f798e-7a76-498a-a553-4bf29eec66b0 kind: Event lastTimestamp: "2026-03-18T16:55:23Z" message: Stopping container switch-graph-a225b metadata: creationTimestamp: "2026-03-18T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:23Z" name: switch-graph-a225b-c4ddc8b65-wjsvf.189dfdd5f3e64e41 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 664e309e-73d3-4021-89b6-4f92a658f218 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T16:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a225b} kind: Pod name: switch-graph-a225b-c4ddc8b65-wjsvf namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 717f798e-7a76-498a-a553-4bf29eec66b0 kind: Event lastTimestamp: "2026-03-18T16:55:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T16:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:55:51Z" name: switch-graph-a225b-c4ddc8b65-wjsvf.189dfdd6a9715a0f namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 513633d9-d071-4454-a23c-8b92ba402d1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-a225b-c4ddc8b65 namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: 5c3f7fab-a2d0-4bbc-9ef3-68f25698c284 kind: Event lastTimestamp: "2026-03-18T16:55:13Z" message: 'Created pod: switch-graph-a225b-c4ddc8b65-wjsvf' metadata: creationTimestamp: "2026-03-18T16:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:13Z" name: switch-graph-a225b-c4ddc8b65.189dfdd3a1abb44a namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: f6238665-dafd-49c5-907e-2bff63e9b178 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-a225b namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 038cb47b-7aa3-49f7-bf01-6a463b532ed9 kind: Event lastTimestamp: "2026-03-18T16:55:13Z" message: Scaled up replica set switch-graph-a225b-c4ddc8b65 from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:55:13Z" name: switch-graph-a225b.189dfdd39fd635f8 namespace: kserve-ci-e2e-test resourceVersion: "14525" uid: 85b8d9d7-720b-429c-9cfe-652fb90bd2ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:55:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-a225b namespace: kserve-ci-e2e-test resourceVersion: "14538" uid: b16d93f2-db74-42e2-9457-59092b5ecb58 kind: Event lastTimestamp: "2026-03-18T16:55:13Z" message: InferenceGraph [switch-graph-a225b] is Ready metadata: creationTimestamp: "2026-03-18T16:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:55:13Z" name: switch-graph-a225b.189dfdd3a517a937 namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: 71ff0883-a703-4994-b2c0-f68fc861c536 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-03-18T16:56:44.039117Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dfe46-d9c5f577d-l5vh7 namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 86ab7c7e-c09f-49e8-9c9c-4ffa6ecc6ab0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-dfe46-d9c5f577d-l5vh7 to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T16:56:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T16:56:44Z" name: switch-graph-dfe46-d9c5f577d-l5vh7.189dfde8ad82f74a namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: 9f8df114-0927-42c9-8d4e-37fd65b64e0c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:44Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dfe46-d9c5f577d-l5vh7 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 86ab7c7e-c09f-49e8-9c9c-4ffa6ecc6ab0 kind: Event lastTimestamp: "2026-03-18T16:56:44Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T16:56:44Z" name: switch-graph-dfe46-d9c5f577d-l5vh7.189dfde8c6b571f1 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: cb6409f0-2d1d-46bf-a084-9986d3dc8097 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dfe46} kind: Pod name: switch-graph-dfe46-d9c5f577d-l5vh7 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 86ab7c7e-c09f-49e8-9c9c-4ffa6ecc6ab0 kind: Event lastTimestamp: "2026-03-18T16:56:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:44Z" name: switch-graph-dfe46-d9c5f577d-l5vh7.189dfde8c7fd3bbc namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 5d2d4bdd-d9a8-4ff6-942b-9c5c77f70aa9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dfe46} kind: Pod name: switch-graph-dfe46-d9c5f577d-l5vh7 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 86ab7c7e-c09f-49e8-9c9c-4ffa6ecc6ab0 kind: Event lastTimestamp: "2026-03-18T16:56:44Z" message: 'Created container: switch-graph-dfe46' metadata: creationTimestamp: "2026-03-18T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:44Z" name: switch-graph-dfe46-d9c5f577d-l5vh7.189dfde8cadfb856 namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 3314f408-c526-49f0-bfe6-1eae2cb1764f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dfe46} kind: Pod name: switch-graph-dfe46-d9c5f577d-l5vh7 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 86ab7c7e-c09f-49e8-9c9c-4ffa6ecc6ab0 kind: Event lastTimestamp: "2026-03-18T16:56:44Z" message: Started container switch-graph-dfe46 metadata: creationTimestamp: "2026-03-18T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T16:56:44Z" name: switch-graph-dfe46-d9c5f577d-l5vh7.189dfde8cb52ad93 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 937e4446-4ce4-4398-b5ed-b06e5020cd9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:04:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dfe46} kind: Pod name: switch-graph-dfe46-d9c5f577d-l5vh7 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 86ab7c7e-c09f-49e8-9c9c-4ffa6ecc6ab0 kind: Event lastTimestamp: "2026-03-18T17:04:58Z" message: Stopping container switch-graph-dfe46 metadata: creationTimestamp: "2026-03-18T17:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:04:58Z" name: switch-graph-dfe46-d9c5f577d-l5vh7.189dfe5bda14201b namespace: kserve-ci-e2e-test resourceVersion: "18675" uid: 682560a0-30bc-4d94-8ab9-7286823c5feb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-03-18T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dfe46} kind: Pod name: switch-graph-dfe46-d9c5f577d-l5vh7 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 86ab7c7e-c09f-49e8-9c9c-4ffa6ecc6ab0 kind: Event lastTimestamp: "2026-03-18T17:05:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:05:25Z" name: switch-graph-dfe46-d9c5f577d-l5vh7.189dfe5c6129a35c namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: 2d22e108-44fe-474c-a9b3-4a2208051e0d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-dfe46-d9c5f577d namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: b2e7deb7-a8cd-4451-8806-5e3ce554d629 kind: Event lastTimestamp: "2026-03-18T16:56:44Z" message: 'Created pod: switch-graph-dfe46-d9c5f577d-l5vh7' metadata: creationTimestamp: "2026-03-18T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:44Z" name: switch-graph-dfe46-d9c5f577d.189dfde8ad1b366a namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 9bcb2838-5e5b-4bbb-95ab-01e44fbd2078 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: 02190a3a-a1c3-4322-a232-5db584981a78 kind: Event lastTimestamp: "2026-03-18T16:56:44Z" message: Scaled up replica set switch-graph-dfe46-d9c5f577d from 0 to 1 metadata: creationTimestamp: "2026-03-18T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:44Z" name: switch-graph-dfe46.189dfde8ac57cf08 namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: da0a0637-286c-4eb7-a5a7-adb287e4a053 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: 170657fd-d884-4618-a1c8-fd8607426d6f kind: Event lastTimestamp: "2026-03-18T16:56:44Z" message: InferenceGraph [switch-graph-dfe46] is Ready metadata: creationTimestamp: "2026-03-18T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T16:56:44Z" name: switch-graph-dfe46.189dfde8afeee0bc namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: cedc6c0b-7355-4d90-802e-5e68f1ae1c14 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 9ff4dda7-5526-4d34-82a4-703dd06b3423 kind: Event lastTimestamp: "2026-03-18T16:56:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:59Z" name: switch-graph-dfe46.189dfdec2ac8c085 namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: e12455e6-371d-4aa6-8c57-1586f32ac98a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:56:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 9ff4dda7-5526-4d34-82a4-703dd06b3423 kind: Event lastTimestamp: "2026-03-18T16:56:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T16:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:56:59Z" name: switch-graph-dfe46.189dfdec2b1ac052 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 2c96f9b5-f731-4e6d-a7bb-c750c0ca2f0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 9ff4dda7-5526-4d34-82a4-703dd06b3423 kind: Event lastTimestamp: "2026-03-18T16:57:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:14Z" name: switch-graph-dfe46.189dfdefa9b0c147 namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: 76752f1c-0aa0-4d3b-be9e-305e58c04efa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T16:57:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dfe46 namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 9ff4dda7-5526-4d34-82a4-703dd06b3423 kind: Event lastTimestamp: "2026-03-18T16:57:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-03-18T16:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T16:57:14Z" name: switch-graph-dfe46.189dfdefa9ef407d namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: 03bdd8a3-e095-4597-a4bb-c23920a6c3aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-03-18T17:18:04.127900Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-eb922-574c9ddc44-2c6pp namespace: kserve-ci-e2e-test resourceVersion: "23894" uid: 1631c823-d67a-417b-a542-fbfbbfb396ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-eb922-574c9ddc44-2c6pp to ip-10-0-143-175.ec2.internal metadata: creationTimestamp: "2026-03-18T17:18:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-03-18T17:18:04Z" name: switch-graph-eb922-574c9ddc44-2c6pp.189dff12b8bfaef1 namespace: kserve-ci-e2e-test resourceVersion: "23899" uid: a1ded620-078b-427c-b914-82fdbcae2108 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-868747f657-hb8fw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:04Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-eb922-574c9ddc44-2c6pp namespace: kserve-ci-e2e-test resourceVersion: "23898" uid: 1631c823-d67a-417b-a542-fbfbbfb396ba kind: Event lastTimestamp: "2026-03-18T17:18:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-eb922-serving-cert" not found' metadata: creationTimestamp: "2026-03-18T17:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:04Z" name: switch-graph-eb922-574c9ddc44-2c6pp.189dff12c1972ec7 namespace: kserve-ci-e2e-test resourceVersion: "23909" uid: 026a13a3-bc7d-4bf5-9333-8f10fe4f90bd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:05Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-eb922-574c9ddc44-2c6pp namespace: kserve-ci-e2e-test resourceVersion: "23903" uid: 1631c823-d67a-417b-a542-fbfbbfb396ba kind: Event lastTimestamp: "2026-03-18T17:18:05Z" message: Add eth0 [10.133.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-03-18T17:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-03-18T17:18:05Z" name: switch-graph-eb922-574c9ddc44-2c6pp.189dff12f67ef021 namespace: kserve-ci-e2e-test resourceVersion: "23919" uid: 2d9da976-879b-4bf5-b191-8c326c107409 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb922} kind: Pod name: switch-graph-eb922-574c9ddc44-2c6pp namespace: kserve-ci-e2e-test resourceVersion: "23898" uid: 1631c823-d67a-417b-a542-fbfbbfb396ba kind: Event lastTimestamp: "2026-03-18T17:18:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af759cb194c6adf55cc603062d0a02c33a0e8792c000ef381a0435f8f6261a76" already present on machine metadata: creationTimestamp: "2026-03-18T17:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:05Z" name: switch-graph-eb922-574c9ddc44-2c6pp.189dff12f7b79db1 namespace: kserve-ci-e2e-test resourceVersion: "23921" uid: 67d226a3-863e-4fe3-a35b-badd6e99a6f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb922} kind: Pod name: switch-graph-eb922-574c9ddc44-2c6pp namespace: kserve-ci-e2e-test resourceVersion: "23898" uid: 1631c823-d67a-417b-a542-fbfbbfb396ba kind: Event lastTimestamp: "2026-03-18T17:18:05Z" message: 'Created container: switch-graph-eb922' metadata: creationTimestamp: "2026-03-18T17:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:05Z" name: switch-graph-eb922-574c9ddc44-2c6pp.189dff12fa9de44e namespace: kserve-ci-e2e-test resourceVersion: "23924" uid: c86462e1-3bf0-4607-8120-74a66ff7634a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb922} kind: Pod name: switch-graph-eb922-574c9ddc44-2c6pp namespace: kserve-ci-e2e-test resourceVersion: "23898" uid: 1631c823-d67a-417b-a542-fbfbbfb396ba kind: Event lastTimestamp: "2026-03-18T17:18:05Z" message: Started container switch-graph-eb922 metadata: creationTimestamp: "2026-03-18T17:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:18:05Z" name: switch-graph-eb922-574c9ddc44-2c6pp.189dff12fb6184ab namespace: kserve-ci-e2e-test resourceVersion: "23925" uid: 15455d05-96d9-4195-a139-afc2965d9554 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:34:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb922} kind: Pod name: switch-graph-eb922-574c9ddc44-2c6pp namespace: kserve-ci-e2e-test resourceVersion: "23898" uid: 1631c823-d67a-417b-a542-fbfbbfb396ba kind: Event lastTimestamp: "2026-03-18T17:34:23Z" message: Stopping container switch-graph-eb922 metadata: creationTimestamp: "2026-03-18T17:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:34:23Z" name: switch-graph-eb922-574c9ddc44-2c6pp.189dfff6bda7a25b namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 8197ce97-876d-41f6-a3b7-b94e3c39bb0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-03-18T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb922} kind: Pod name: switch-graph-eb922-574c9ddc44-2c6pp namespace: kserve-ci-e2e-test resourceVersion: "23898" uid: 1631c823-d67a-417b-a542-fbfbbfb396ba kind: Event lastTimestamp: "2026-03-18T17:34:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-03-18T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-03-18T17:34:47Z" name: switch-graph-eb922-574c9ddc44-2c6pp.189dfff7998007de namespace: kserve-ci-e2e-test resourceVersion: "28524" uid: 5740db96-cfb9-4307-9473-24739c9a7c97 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-175.ec2.internal source: component: kubelet host: ip-10-0-143-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-eb922-574c9ddc44 namespace: kserve-ci-e2e-test resourceVersion: "23891" uid: b0c2215d-84b1-43d6-83ea-c85491d3b9f1 kind: Event lastTimestamp: "2026-03-18T17:18:04Z" message: 'Created pod: switch-graph-eb922-574c9ddc44-2c6pp' metadata: creationTimestamp: "2026-03-18T17:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:04Z" name: switch-graph-eb922-574c9ddc44.189dff12b85eb0be namespace: kserve-ci-e2e-test resourceVersion: "23896" uid: 57ea1a98-201d-46eb-ba7c-f2c03a2ba00c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-eb922 namespace: kserve-ci-e2e-test resourceVersion: "23889" uid: c06c4b9d-59b3-4b58-8310-ebed581a6ad5 kind: Event lastTimestamp: "2026-03-18T17:18:04Z" message: Scaled up replica set switch-graph-eb922-574c9ddc44 from 0 to 1 metadata: creationTimestamp: "2026-03-18T17:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:04Z" name: switch-graph-eb922.189dff12b77c5944 namespace: kserve-ci-e2e-test resourceVersion: "23892" uid: b1d170a5-ce93-4cda-b9ba-d98be902c611 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-03-18T17:18:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-eb922 namespace: kserve-ci-e2e-test resourceVersion: "23905" uid: 88eb150a-9914-499d-b0f8-bb76b62224a2 kind: Event lastTimestamp: "2026-03-18T17:18:04Z" message: InferenceGraph [switch-graph-eb922] is Ready metadata: creationTimestamp: "2026-03-18T17:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-03-18T17:18:04Z" name: switch-graph-eb922.189dff12bad346a9 namespace: kserve-ci-e2e-test resourceVersion: "23907" uid: cde815c8-4431-4221-9ed3-74b124860c08 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:18:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-eb922 namespace: kserve-ci-e2e-test resourceVersion: "23890" uid: 9b1921d0-d562-46d9-85f9-c7ea0a95840f kind: Event lastTimestamp: "2026-03-18T17:18:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:34Z" name: switch-graph-eb922.189dff1635f93f94 namespace: kserve-ci-e2e-test resourceVersion: "24084" uid: 4f62fc56-37cc-4c87-9bf1-e88000fffe17 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-03-18T17:18:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-eb922 namespace: kserve-ci-e2e-test resourceVersion: "23890" uid: 9b1921d0-d562-46d9-85f9-c7ea0a95840f kind: Event lastTimestamp: "2026-03-18T17:18:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-03-18T17:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-03-18T17:18:34Z" name: switch-graph-eb922.189dff16365645b5 namespace: kserve-ci-e2e-test resourceVersion: "24085" uid: 372aad8b-c878-43b1-9263-16bbfd71fa6e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "28526"