--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:37:36.538107Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-8dddc-6dffc4bb86-lq88m namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 0e17e79b-772b-4ba5-873a-16336560d1bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-8dddc-6dffc4bb86-lq88m to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:37:36Z" name: ensemble-graph-8dddc-6dffc4bb86-lq88m.18c8f963290bc5fd namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: a72dcaa2-cb62-4cd4-ab1c-66aac64fd8da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-8dddc-6dffc4bb86-lq88m namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: 0e17e79b-772b-4ba5-873a-16336560d1bc kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:37:36Z" name: ensemble-graph-8dddc-6dffc4bb86-lq88m.18c8f96342dad695 namespace: kserve-ci-e2e-test resourceVersion: "21866" uid: 59946f94-6336-4b30-9192-ff009f300d85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8dddc} kind: Pod name: ensemble-graph-8dddc-6dffc4bb86-lq88m namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 0e17e79b-772b-4ba5-873a-16336560d1bc kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:36Z" name: ensemble-graph-8dddc-6dffc4bb86-lq88m.18c8f96344410798 namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: 0560a03f-6760-410a-aa54-9aa986b2eb46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8dddc} kind: Pod name: ensemble-graph-8dddc-6dffc4bb86-lq88m namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 0e17e79b-772b-4ba5-873a-16336560d1bc kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: 'Created container: ensemble-graph-8dddc' metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: ensemble-graph-8dddc-6dffc4bb86-lq88m.18c8f9634721363f namespace: kserve-ci-e2e-test resourceVersion: "21869" uid: aa7deba8-04aa-4f5b-9b6f-8f224a0e0ece reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8dddc} kind: Pod name: ensemble-graph-8dddc-6dffc4bb86-lq88m namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 0e17e79b-772b-4ba5-873a-16336560d1bc kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Started container ensemble-graph-8dddc metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: ensemble-graph-8dddc-6dffc4bb86-lq88m.18c8f963478c321c namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: 8921b96b-3e6d-411a-a6b5-dae54d0b1974 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8dddc} kind: Pod name: ensemble-graph-8dddc-6dffc4bb86-lq88m namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 0e17e79b-772b-4ba5-873a-16336560d1bc kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Stopping container ensemble-graph-8dddc metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: ensemble-graph-8dddc-6dffc4bb86-lq88m.18c8f9d650462d62 namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: a752de8d-0b86-4d90-ab63-46ab8ef62c25 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8dddc} kind: Pod name: ensemble-graph-8dddc-6dffc4bb86-lq88m namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 0e17e79b-772b-4ba5-873a-16336560d1bc kind: Event lastTimestamp: "2026-08-05T17:46:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:18Z" name: ensemble-graph-8dddc-6dffc4bb86-lq88m.18c8f9d6d2576626 namespace: kserve-ci-e2e-test resourceVersion: "26052" uid: 90713e63-6d54-4d3a-afbd-189df4dd4c38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-8dddc-6dffc4bb86 namespace: kserve-ci-e2e-test resourceVersion: "21787" uid: 65cf740b-a0a6-471c-8f1c-c36a2d5626dd kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'Created pod: ensemble-graph-8dddc-6dffc4bb86-lq88m' metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:36Z" name: ensemble-graph-8dddc-6dffc4bb86.18c8f963289a18bd namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: ecf64c1f-5302-4d65-b650-086eb9f90869 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: f61ebb5f-7b45-4652-8282-ace65311c8fb kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Scaled up replica set ensemble-graph-8dddc-6dffc4bb86 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:36Z" name: ensemble-graph-8dddc.18c8f96327e13471 namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 3ad3e489-423d-4732-8b01-5bc9eab32e71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 7d5df1ba-9272-4aeb-8c5b-ab4a4db874d8 kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: InferenceGraph [ensemble-graph-8dddc] is Ready metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:36Z" name: ensemble-graph-8dddc.18c8f9632ccd328d namespace: kserve-ci-e2e-test resourceVersion: "21838" uid: c47b1639-e04a-4a9e-bbd6-06a6f9ed2a87 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: 49af3785-0d97-4c34-bb95-9e3e710d1424 kind: Event lastTimestamp: "2026-08-05T17:38:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:06Z" name: ensemble-graph-8dddc.18c8f966a67a5495 namespace: kserve-ci-e2e-test resourceVersion: "22145" uid: 64d6f871-e7da-4d0d-8620-58f28295b9cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: 49af3785-0d97-4c34-bb95-9e3e710d1424 kind: Event lastTimestamp: "2026-08-05T17:38:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:06Z" name: ensemble-graph-8dddc.18c8f966a6ecf214 namespace: kserve-ci-e2e-test resourceVersion: "22147" uid: 1dcd21c7-2ef3-4e08-b3ba-7bd9a3b6806b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:36:26.240505Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fba55-75554f9f4b-9xzw2 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: 0b4456d4-3d9f-4147-be8d-448129f3d2f3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-fba55-75554f9f4b-9xzw2 to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:36:26Z" name: ensemble-graph-fba55-75554f9f4b-9xzw2.18c8f952cafb65ed namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: a3e1fd49-78a4-4ad7-8e7e-e2cfd1e0f7c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fba55-75554f9f4b-9xzw2 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 0b4456d4-3d9f-4147-be8d-448129f3d2f3 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:36:26Z" name: ensemble-graph-fba55-75554f9f4b-9xzw2.18c8f952e4787d67 namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: a2878e01-0167-47f7-94d7-61f4baae1b36 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fba55} kind: Pod name: ensemble-graph-fba55-75554f9f4b-9xzw2 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 0b4456d4-3d9f-4147-be8d-448129f3d2f3 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: ensemble-graph-fba55-75554f9f4b-9xzw2.18c8f952e5fe15ca namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 08d0f9c1-376d-45b1-aa06-e368c2c0e664 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fba55} kind: Pod name: ensemble-graph-fba55-75554f9f4b-9xzw2 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 0b4456d4-3d9f-4147-be8d-448129f3d2f3 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'Created container: ensemble-graph-fba55' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: ensemble-graph-fba55-75554f9f4b-9xzw2.18c8f952e9daa476 namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: 7a1b355c-1211-44a2-8115-83a9be963d7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fba55} kind: Pod name: ensemble-graph-fba55-75554f9f4b-9xzw2 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 0b4456d4-3d9f-4147-be8d-448129f3d2f3 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Started container ensemble-graph-fba55 metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: ensemble-graph-fba55-75554f9f4b-9xzw2.18c8f952ea4b8bd3 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 82378663-a3bd-4dc5-a5a0-f29b624aacbf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fba55} kind: Pod name: ensemble-graph-fba55-75554f9f4b-9xzw2 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 0b4456d4-3d9f-4147-be8d-448129f3d2f3 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: Stopping container ensemble-graph-fba55 metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:36Z" name: ensemble-graph-fba55-75554f9f4b-9xzw2.18c8f95521541179 namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: a105a9bd-96c6-4215-aaa5-030700d7f2f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:36:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fba55} kind: Pod name: ensemble-graph-fba55-75554f9f4b-9xzw2 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 0b4456d4-3d9f-4147-be8d-448129f3d2f3 kind: Event lastTimestamp: "2026-08-05T17:37:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:03Z" name: ensemble-graph-fba55-75554f9f4b-9xzw2.18c8f95587ee5ccd namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 6f179dfc-dd49-4193-9d46-2908a057f267 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-fba55-75554f9f4b namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: 448b941b-4517-4bd3-bd16-db8e231d7d0d kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'Created pod: ensemble-graph-fba55-75554f9f4b-9xzw2' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: ensemble-graph-fba55-75554f9f4b.18c8f952ca219153 namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: 3a501f29-f0ac-4dad-8e5c-7069359920fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-fba55 namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 9d219a43-7ef8-4f12-8343-4250679f06a2 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Scaled up replica set ensemble-graph-fba55-75554f9f4b from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: ensemble-graph-fba55.18c8f952c93c78c0 namespace: kserve-ci-e2e-test resourceVersion: "20844" uid: 3c5a1a2f-f234-4e46-a86d-28c4ed29e0d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-fba55 namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: 89ad2eeb-a2a6-4953-8ad5-d7225ee87c4c kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: InferenceGraph [ensemble-graph-fba55] is Ready metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:26Z" name: ensemble-graph-fba55.18c8f952cee1719c namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: f89aa56a-ecd6-44c5-9d02-80142ab413c4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:36:26.275630Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-27640-predictor-799cfccbdc-v8s6m to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f952cd137498 namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: ffb55cc9-8e86-47f1-a618-5abcbec2bae5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f952e677fce9 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 4d2fb3df-9645-4adc-8321-4b249e52ac1f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f952e8399874 namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: bcd2ba89-66a2-4e1d-baea-a70646f08e6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f952eaf6b00b namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: 7008796c-a5c4-4cbc-a342-bacd5c7701d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f952eb65c291 namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: 8482213e-84a0-43d9-9a8f-4970e4269c3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f952eb7038c6 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: bb0a906e-46ac-419d-9cf6-8e54aeed469f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f952ee35fb25 namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: 26f30425-290f-4da1-8b2e-af25ffe8a7b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f952eea45d86 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 9a7d0bf5-e726-4a20-b5d5-32e88aea3765 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:37:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:12Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f952f2e57dfb namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: 24f1303a-0241-4abe-a459-e5fe2897c8e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:36Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f9632c16dd51 namespace: kserve-ci-e2e-test resourceVersion: "21830" uid: 37d20823-1f41-494f-a929-7028a436c53d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:36Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f9632c174efa namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 4c2b98ba-731c-4843-ad07-535469d38064 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 2dbefbae-0497-4396-ab82-abcd1a08abc2 kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: error-404-isvc-27640-predictor-799cfccbdc-v8s6m.18c8f9637ba25a1a namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: a70bb99a-9fd5-45f4-9ff1-66ab297db021 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-27640-predictor-799cfccbdc namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: c19ec60f-3ff2-4cb1-a59a-861ef410d2fd kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'Created pod: error-404-isvc-27640-predictor-799cfccbdc-v8s6m' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor-799cfccbdc.18c8f952cc99e367 namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: 87e84570-cf61-44c0-ad32-fb0fd0cefe52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: 6c7f708c-6db1-46b7-b94b-d40512b6adb6 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Scaled up replica set error-404-isvc-27640-predictor-799cfccbdc from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640-predictor.18c8f952ca59150f namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 9c0693b9-9423-4bb5-8fe4-1afb6b0561bd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: c577d859-38c7-4efb-93c5-34973b98cab7 kind: Event lastTimestamp: "2026-08-05T17:36:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:41Z" name: error-404-isvc-27640-predictor.18c8f9564921c383 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 43d5c3f8-02cf-4bb3-80b1-390be5f22cd2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: c577d859-38c7-4efb-93c5-34973b98cab7 kind: Event lastTimestamp: "2026-08-05T17:36:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:41Z" name: error-404-isvc-27640-predictor.18c8f956498b0d56 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: c0b56dc3-79e8-4f71-965a-2f09499c9df5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:36:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: c577d859-38c7-4efb-93c5-34973b98cab7 kind: Event lastTimestamp: "2026-08-05T17:37:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:26Z" name: error-404-isvc-27640-predictor.18c8f959c8f7a8e1 namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: 511502fc-5a59-4379-8ea6-2c2ddbbc926a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:36:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: c577d859-38c7-4efb-93c5-34973b98cab7 kind: Event lastTimestamp: "2026-08-05T17:37:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:26Z" name: error-404-isvc-27640-predictor.18c8f959ca0f3162 namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 9f73aa26-77a6-48f9-9bc9-e8c6522398f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-27640 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: ae3c787b-bcd2-49db-bc3d-e6b780e1e6b3 kind: Event lastTimestamp: "2026-08-05T17:37:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:23Z" name: error-404-isvc-27640.18c8f952cafb4226 namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: cb4b0b76-8251-42db-a44b-e1ee3f67e3dc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-27640 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: ae3c787b-bcd2-49db-bc3d-e6b780e1e6b3 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-27640": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-27640": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640.18c8f952cbde01d2 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 3e0a9312-a42d-4e7b-8479-f4b1e2dd1e36 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-27640 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: ae3c787b-bcd2-49db-bc3d-e6b780e1e6b3 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-27640": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-27640.18c8f952cbde7067 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: 43b33c22-0aba-43eb-a5f0-a4e7dab9852b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-27640 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: ae3c787b-bcd2-49db-bc3d-e6b780e1e6b3 kind: Event lastTimestamp: "2026-08-05T17:37:23Z" message: InferenceService [error-404-isvc-27640] is Ready metadata: creationTimestamp: "2026-08-05T17:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:23Z" name: error-404-isvc-27640.18c8f96006e546ee namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 19fe4871-db88-4629-a4e5-e345af0fd92f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:46:51.446116Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26436" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-741a0-predictor-6c788cfb87-2t5vl to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e45c23f7ef namespace: kserve-ci-e2e-test resourceVersion: "26440" uid: 1677f636-d338-4444-9948-7fd3d27b7f44 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-741a0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e4679cd9d6 namespace: kserve-ci-e2e-test resourceVersion: "26462" uid: b779d35a-2edb-407a-8ed0-d8066ecff65f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T17:46:52Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:46:52Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e499682e12 namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: adf97e98-a9e3-42a6-b612-83c0987c9dae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T17:46:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:52Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e49b039801 namespace: kserve-ci-e2e-test resourceVersion: "26487" uid: 0924d378-16b0-4b28-8322-d794ad357c5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T17:46:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:52Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e49e76d405 namespace: kserve-ci-e2e-test resourceVersion: "26489" uid: 57779394-4b79-4252-b2a1-6ba5597da92f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T17:46:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:52Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e49ee2cf4c namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: 923cf64f-6bca-4abf-a4e7-937c1a6dc8fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T17:46:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:52Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e49eec52cf namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: 673c5899-b78f-47dc-b11b-46b0468bd131 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T17:46:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:52Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e4a204cab3 namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: 4d7899d8-bb12-4a72-8c87-5b5a938f1a2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T17:46:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:52Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e4a274d177 namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: 1a6e448b-a34d-4872-ad39-dedc80ba75eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T17:47:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:30Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8f9e519c65d7a namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: ca07c807-c474-47b6-b9ad-9e0bdc970fea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T18:04:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T18:04:20Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8fad8a9dedbc8 namespace: kserve-ci-e2e-test resourceVersion: "34162" uid: eed0765a-eabc-4da0-98e5-97552e310e3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 04ea2238-8ad7-4bc1-9021-e17922673b58 kind: Event lastTimestamp: "2026-08-05T18:04:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T18:04:20Z" name: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl.18c8fad8a9df67f0 namespace: kserve-ci-e2e-test resourceVersion: "34163" uid: b6ad67e3-b575-4003-a494-120f483cb66a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-741a0-predictor-6c788cfb87 namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: b8ed3f8c-a5a0-496b-987d-a466996ca3c0 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'Created pod: error-404-isvc-741a0-predictor-6c788cfb87-2t5vl' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-741a0-predictor-6c788cfb87.18c8f9e45b96c10e namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: dd37edfd-b54d-4b23-aeb0-e7a32687bd62 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 602f1525-a5e9-4e8c-8ea4-ce5b40aca936 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Scaled up replica set error-404-isvc-741a0-predictor-6c788cfb87 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-741a0-predictor.18c8f9e459b6d616 namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: cc49402f-ae3d-4167-86a5-9fedd90f971c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:47:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: eef251e9-779e-4170-b92c-91c613bc5f0e kind: Event lastTimestamp: "2026-08-05T17:47:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:21Z" name: error-404-isvc-741a0-predictor.18c8f9e7d846f49b namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: 48d320c1-dc2f-444e-af6a-1eca4b20b098 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:47:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: eef251e9-779e-4170-b92c-91c613bc5f0e kind: Event lastTimestamp: "2026-08-05T17:47:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:21Z" name: error-404-isvc-741a0-predictor.18c8f9e7d8bb0174 namespace: kserve-ci-e2e-test resourceVersion: "26901" uid: e0f2b4a8-cc84-4d61-9567-e53d072cf677 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:47:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26770" uid: eef251e9-779e-4170-b92c-91c613bc5f0e kind: Event lastTimestamp: "2026-08-05T17:48:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:06Z" name: error-404-isvc-741a0-predictor.18c8f9eed6681316 namespace: kserve-ci-e2e-test resourceVersion: "27360" uid: 8441c0ba-f4ce-4624-b574-759475312858 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:47:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26770" uid: eef251e9-779e-4170-b92c-91c613bc5f0e kind: Event lastTimestamp: "2026-08-05T17:48:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:06Z" name: error-404-isvc-741a0-predictor.18c8f9eed6df6560 namespace: kserve-ci-e2e-test resourceVersion: "27361" uid: 63ed916d-0618-4ff5-a040-161d751be207 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-741a0 namespace: kserve-ci-e2e-test resourceVersion: "26391" uid: 32640929-8841-42d8-b09c-f3ef0595ddb0 kind: Event lastTimestamp: "2026-08-05T17:47:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:40Z" name: error-404-isvc-741a0.18c8f9e459b9daee namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 4434686c-c295-46c3-8e53-1cb5171d411b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-741a0 namespace: kserve-ci-e2e-test resourceVersion: "26391" uid: 32640929-8841-42d8-b09c-f3ef0595ddb0 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-741a0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-741a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-741a0.18c8f9e45a93e2b5 namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: 5f667acc-c9bc-495c-a5e5-297a27203cf1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-741a0 namespace: kserve-ci-e2e-test resourceVersion: "26391" uid: 32640929-8841-42d8-b09c-f3ef0595ddb0 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-741a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-741a0.18c8f9e45a942848 namespace: kserve-ci-e2e-test resourceVersion: "26435" uid: 5ff2ecc3-2cc6-4a04-9ba5-8120b6090461 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-741a0 namespace: kserve-ci-e2e-test resourceVersion: "27067" uid: 32640929-8841-42d8-b09c-f3ef0595ddb0 kind: Event lastTimestamp: "2026-08-05T17:47:40Z" message: InferenceService [error-404-isvc-741a0] is Ready metadata: creationTimestamp: "2026-08-05T17:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:40Z" name: error-404-isvc-741a0.18c8f9efd5cec03c namespace: kserve-ci-e2e-test resourceVersion: "27068" uid: d848d95b-7c91-401d-b497-2461ff3820e4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:24:41.381801Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-829a9-predictor-67f4589d75-vnfjn to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:24:41Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8aeae212e5a namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: df06b48e-aa12-49c7-96ea-dc2e37441aff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:24:41Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8aec7808730 namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: c6cb3f65-2b37-4326-843b-663a7c34f963 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:41Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8aec8ce1831 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: 30c384ea-3c4f-4af4-9d3a-0a5e129f0b7c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:24:57Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" in 15.821s (15.821s including waiting). Image size: 1333945440 bytes.' metadata: creationTimestamp: "2026-08-05T17:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:57Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8b277dcbd45 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 6a7f04b7-a8ec-4a4c-a11e-305391194049 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:24:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:57Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8b27beef908 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: ca9fad3b-caa5-4f3b-b48a-a66e3ef4531c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:24:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:57Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8b27c5df0f2 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 10d62aab-79c6-43d8-81ae-064c428de8a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:24:57Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-08-05T17:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:57Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8b27c6b4460 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: e539d1a6-a8cb-4ec0-b0dd-b5358751b5c8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:25:00Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.543s (2.543s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-08-05T17:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:00Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8b3140542a4 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: c2307b1d-29fe-4572-b8bd-04fe20745dc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:25:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:00Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8b317c19656 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: b62cfd00-615d-4fdf-b578-8fe1b8cf9193 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:25:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:00Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8b3183fbbb8 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 6d0b28a5-714d-487a-b8a7-8a562cb5f06d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:25:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:25:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:37Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8b3400955ec namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 3268a066-134f-42fc-a073-6630365767d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8c39e646e4b namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: 5c710b13-b7bd-41bc-ad62-da73f690c055 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8c39e678ae9 namespace: kserve-ci-e2e-test resourceVersion: "15793" uid: 6a5b7684-3b03-4e8f-85a2-eb6f3985b86a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 419c1eaa-2d46-4226-94b6-9d1bbe38e8f0 kind: Event lastTimestamp: "2026-08-05T17:26:12Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:12Z" name: error-404-isvc-829a9-predictor-67f4589d75-vnfjn.18c8f8c3c8364f5e namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: d6fefcf3-a171-470f-8f3c-60618feca759 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-829a9-predictor-67f4589d75 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: d9c6b3d9-3eba-4966-8dba-70ff11e65f3a kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: 'Created pod: error-404-isvc-829a9-predictor-67f4589d75-vnfjn' metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:41Z" name: error-404-isvc-829a9-predictor-67f4589d75.18c8f8aead9ebb79 namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: cbca3a39-1f69-4776-92e8-b69ed77bb203 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: cb876ba3-80d2-4afb-bb18-6e9eb253bcd6 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Scaled up replica set error-404-isvc-829a9-predictor-67f4589d75 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:41Z" name: error-404-isvc-829a9-predictor.18c8f8aeabb3e52f namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 300f8bec-2553-4b72-8c73-755f05882022 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 3b547def-60ed-40d6-9758-7c58901f5bc5 kind: Event lastTimestamp: "2026-08-05T17:25:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:26Z" name: error-404-isvc-829a9-predictor.18c8f8b22a85035a namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: c6ab034a-e12d-49b6-88f3-23c7563ce68f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 3b547def-60ed-40d6-9758-7c58901f5bc5 kind: Event lastTimestamp: "2026-08-05T17:25:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:26Z" name: error-404-isvc-829a9-predictor.18c8f8b22af6bbee namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 8b847573-0a5d-4364-b6c8-757a8b945dd5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:25:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: 3b547def-60ed-40d6-9758-7c58901f5bc5 kind: Event lastTimestamp: "2026-08-05T17:25:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:56Z" name: error-404-isvc-829a9-predictor.18c8f8bca758e499 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 3f0190ae-68c7-4cff-bc2e-669c2ad32d74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:25:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: 3b547def-60ed-40d6-9758-7c58901f5bc5 kind: Event lastTimestamp: "2026-08-05T17:25:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:56Z" name: error-404-isvc-829a9-predictor.18c8f8bca7ce638a namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 50991f56-2fc1-48b1-9ead-58ff4a673e28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-829a9 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 0a0ce07b-bf44-44a3-9e8f-9cd54e87a971 kind: Event lastTimestamp: "2026-08-05T17:25:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:25:47Z" name: error-404-isvc-829a9.18c8f8aeabe1241f namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: a6389990-2dd5-4209-bf6c-67b0afb23c49 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-829a9 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: 0a0ce07b-bf44-44a3-9e8f-9cd54e87a971 kind: Event lastTimestamp: "2026-08-05T17:25:47Z" message: InferenceService [error-404-isvc-829a9] is Ready metadata: creationTimestamp: "2026-08-05T17:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:25:47Z" name: error-404-isvc-829a9.18c8f8be0229d416 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: f67c5962-7f55-4e3f-a93f-93c1fa65087f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:36:36.708354Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8dddc-predictor-5558b5787-9qrcb to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:36:36Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f9553aea2f4d namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 1401e4d8-b73b-4092-9ff0-2a2b1f156f32 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:36:37Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f95554cf4901 namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: 21196bae-d414-4d59-a341-357e005bb3de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f955564d7b09 namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: b5b69dcf-f734-4a81-96ff-48314cd27185 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f955590b96c2 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: ba0602d7-8554-4d5d-a473-10e7c3f77dda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f955597a1f0a namespace: kserve-ci-e2e-test resourceVersion: "21144" uid: 52da1d3c-0d50-4f1e-a056-94dd44ac6fb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f95559842b14 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: b4843f4e-2274-4a77-a3d1-d99e180de4d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f9555d718f25 namespace: kserve-ci-e2e-test resourceVersion: "21146" uid: 8e0c0bb2-23d3-47f4-bf55-54e8b8a3c980 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f9555e2a7d54 namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 6e0c5564-ced2-424a-8fe0-c4eebc359e0e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:45:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:54Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f95586dfbea6 namespace: kserve-ci-e2e-test resourceVersion: "25854" uid: d343e4ca-5316-4063-9615-3e91c509177d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f9d66330466f namespace: kserve-ci-e2e-test resourceVersion: "25780" uid: eb6d7cbd-f431-46bf-9b63-2eaddcadfd61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f9d66330a7f0 namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: d2426d63-df7e-404e-9e6e-6d9c1e3c99ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: e8b2fd82-37b5-457e-ae17-47fadcce2a5e kind: Event lastTimestamp: "2026-08-05T17:45:53Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:53Z" name: error-404-isvc-8dddc-predictor-5558b5787-9qrcb.18c8f9d6fb4fb3db namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: b66ea82d-a9bd-473a-9110-09a58de60b9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8dddc-predictor-5558b5787 namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: b4e76c03-ed02-41c4-85dc-cd78c5bdb9c6 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: 'Created pod: error-404-isvc-8dddc-predictor-5558b5787-9qrcb' metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:36Z" name: error-404-isvc-8dddc-predictor-5558b5787.18c8f9553a7bba67 namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: 8518ae0c-46d6-4a8d-a95d-d56c3f47228d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: b7251f4f-2943-489a-808c-96886a340c4a kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: Scaled up replica set error-404-isvc-8dddc-predictor-5558b5787 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:36Z" name: error-404-isvc-8dddc-predictor.18c8f95535485d7e namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 9b10ef2e-f422-4ac1-9e2d-9fa7f9c6f558 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:36:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 0cb84455-ec80-46c4-b3b2-e015a840fc9c kind: Event lastTimestamp: "2026-08-05T17:38:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:06Z" name: error-404-isvc-8dddc-predictor.18c8f958b3f2362a namespace: kserve-ci-e2e-test resourceVersion: "22150" uid: 84edabb6-387e-40a8-81d4-95859422ae64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:36:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 0cb84455-ec80-46c4-b3b2-e015a840fc9c kind: Event lastTimestamp: "2026-08-05T17:38:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:06Z" name: error-404-isvc-8dddc-predictor.18c8f958b457b8da namespace: kserve-ci-e2e-test resourceVersion: "22151" uid: 984ce40f-7a58-4309-81c7-beaa1a9c6021 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21291" uid: 0cb84455-ec80-46c4-b3b2-e015a840fc9c kind: Event lastTimestamp: "2026-08-05T17:37:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:06Z" name: error-404-isvc-8dddc-predictor.18c8f95c33319569 namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: 1eb96980-e08a-4258-b9c1-16bb21cd95f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21291" uid: 0cb84455-ec80-46c4-b3b2-e015a840fc9c kind: Event lastTimestamp: "2026-08-05T17:37:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:06Z" name: error-404-isvc-8dddc-predictor.18c8f95c33c7748b namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 8984445d-69c8-4193-98d2-39ee94c1b2e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: ac247cca-a502-4dbd-b5ce-08cda5547bf1 kind: Event lastTimestamp: "2026-08-05T17:37:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:34Z" name: error-404-isvc-8dddc.18c8f95535621d2b namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: 22088cb3-c1ba-42c8-b62f-7994bc523526 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: ac247cca-a502-4dbd-b5ce-08cda5547bf1 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-8dddc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8dddc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:36Z" name: error-404-isvc-8dddc.18c8f95535f02f3f namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 6748ea22-94c6-42d5-aaaa-073003c2dab1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: ac247cca-a502-4dbd-b5ce-08cda5547bf1 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8dddc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:36Z" name: error-404-isvc-8dddc.18c8f95535f08b4e namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 345b35c1-cc31-4da8-ba72-e9ab7233be9d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21692" uid: ac247cca-a502-4dbd-b5ce-08cda5547bf1 kind: Event lastTimestamp: "2026-08-05T17:37:34Z" message: InferenceService [error-404-isvc-8dddc] is Ready metadata: creationTimestamp: "2026-08-05T17:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:34Z" name: error-404-isvc-8dddc.18c8f96296d6c032 namespace: kserve-ci-e2e-test resourceVersion: "21693" uid: 087cdb63-5a52-4904-a65c-9e085c855a28 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:45:51.446403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25779" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-92074-predictor-589dc7f869-8mmn2 to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:45:51Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d663e10c80 namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: 3a2e68cc-d394-4c62-9691-01c73d388543 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-92074-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d6709bf6db namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: d2af5a8a-216c-49ef-9eab-66d95b241130 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:45:52Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:45:52Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d6a17f9d43 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 838a6f7c-f8be-41d0-8c62-c8d76dd97390 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:45:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:52Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d6a311890b namespace: kserve-ci-e2e-test resourceVersion: "25829" uid: 7e775b19-c1ec-46b0-9320-c927edc7094a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:45:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:52Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d6a65bebe8 namespace: kserve-ci-e2e-test resourceVersion: "25832" uid: bf6f4867-1672-4dad-8e79-005444303102 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:45:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:52Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d6a6cedb8a namespace: kserve-ci-e2e-test resourceVersion: "25833" uid: 43948bab-61b1-4c49-a87c-094640c61bc3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:45:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:52Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d6a6d8ebcc namespace: kserve-ci-e2e-test resourceVersion: "25834" uid: da096b22-e0b9-49ec-b6d2-fadb0fe9e5ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:45:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:52Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d6a9e1f7f4 namespace: kserve-ci-e2e-test resourceVersion: "25835" uid: 72ac5792-52e5-4d7b-9284-517e7166db05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:45:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:52Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d6aa50802a namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: 21dea046-aea2-4963-ba1a-b691b4bcd607 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:45:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:46:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:30Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9d70fb025e9 namespace: kserve-ci-e2e-test resourceVersion: "26146" uid: ca48aaf9-220d-40d8-a6da-f0ab0b835d64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:01Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9e6bfd901b2 namespace: kserve-ci-e2e-test resourceVersion: "26664" uid: 69a9ec5e-0af8-4348-b792-7fcc542ef85d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92074-predictor-589dc7f869-8mmn2 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 45df6046-112b-493c-b9bf-b746d05c3a21 kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:01Z" name: error-404-isvc-92074-predictor-589dc7f869-8mmn2.18c8f9e6bfd9a080 namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: 9ce8cd82-3db8-440d-8134-f0d33983e392 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-92074-predictor-589dc7f869 namespace: kserve-ci-e2e-test resourceVersion: "25760" uid: 0ca9c928-f7b5-4acc-b596-0ce198104efc kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: 'Created pod: error-404-isvc-92074-predictor-589dc7f869-8mmn2' metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:45:51Z" name: error-404-isvc-92074-predictor-589dc7f869.18c8f9d66336f27a namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: 5ab80ebe-9e6a-4144-9a68-f43eafd6c3b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: d0cfb9e2-069a-4434-bdb8-1fb05b232e0c kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Scaled up replica set error-404-isvc-92074-predictor-589dc7f869 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:45:51Z" name: error-404-isvc-92074-predictor.18c8f9d6612413ec namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: cad6331d-0780-4fec-9de1-1ee91c7f5370 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:46:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: 32297069-c60d-4fad-9527-42d4fe67f8c4 kind: Event lastTimestamp: "2026-08-05T17:46:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:36Z" name: error-404-isvc-92074-predictor.18c8f9d9dfbef544 namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 3e27137d-69fa-4a56-b2ea-aa03c0decd6d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:46:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: 32297069-c60d-4fad-9527-42d4fe67f8c4 kind: Event lastTimestamp: "2026-08-05T17:46:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:36Z" name: error-404-isvc-92074-predictor.18c8f9d9e03d478d namespace: kserve-ci-e2e-test resourceVersion: "26189" uid: cd694483-9b31-47a9-8c47-c5a0e1104bb5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: 32297069-c60d-4fad-9527-42d4fe67f8c4 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-92074-predictor.18c8f9e45ca036a3 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: ceac1bc2-af05-4599-be53-0a9ad3616c62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: 32297069-c60d-4fad-9527-42d4fe67f8c4 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-92074-predictor.18c8f9e45d22de8f namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: db5bf152-b892-4a88-b239-7de2f6828c81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92074 namespace: kserve-ci-e2e-test resourceVersion: "25738" uid: 319ede41-0836-401b-b523-d8aa1c50b687 kind: Event lastTimestamp: "2026-08-05T17:46:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:46:40Z" name: error-404-isvc-92074.18c8f9d66143b9c5 namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: 9617d8f4-b463-49dc-b609-7422fa866288 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92074 namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: 319ede41-0836-401b-b523-d8aa1c50b687 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-92074": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-92074": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:45:51Z" name: error-404-isvc-92074.18c8f9d66bff585b namespace: kserve-ci-e2e-test resourceVersion: "25797" uid: d577386f-8d7a-4eda-a760-bd2a08fab433 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92074 namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: 319ede41-0836-401b-b523-d8aa1c50b687 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-92074": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:45:51Z" name: error-404-isvc-92074.18c8f9d66bffa699 namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 7968e5b2-f9b7-4bcf-85be-d0a9f1a7805e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92074 namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: 319ede41-0836-401b-b523-d8aa1c50b687 kind: Event lastTimestamp: "2026-08-05T17:46:40Z" message: InferenceService [error-404-isvc-92074] is Ready metadata: creationTimestamp: "2026-08-05T17:46:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:46:40Z" name: error-404-isvc-92074.18c8f9e1cc2246b5 namespace: kserve-ci-e2e-test resourceVersion: "26234" uid: 4d3dc159-f9dd-4c98-a091-905cab22e4c7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:26:11.413165Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f8c3a46ab9ae namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: 34a7d667-f090-4e08-ab1c-ecd7ffafb2c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f8c3bdb7bfb7 namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 4a859b97-be26-4db8-961d-3ac9707c8653 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f8c3bf1cc24b namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: f2c95a55-a62c-4fa4-8012-d3e5dbcc16a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f8c3c20aa9fb namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 60031e0d-6e25-4163-a674-9da36a00ba9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f8c3c27906a1 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: be8c458a-2428-432a-9eb3-203cec9538a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f8c3c2821cac namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 1c128b62-2d7e-4d1e-a694-ee4fc4219eb1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f8c3c4f7affe namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: b18563ca-a2bd-43e4-97af-dcb5beac72e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f8c3c562f53f namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 833500f7-9bc8-473c-bb81-6990fdfa36f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:35:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:28Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f8c3da6eac87 namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: cd23832b-5d30-4340-bfa9-928e6c31e810 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f944d4cbc64e namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: c8a0b2e9-3b45-4e0c-9da8-097196b82b35 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f944d4cc4a5e namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: cd80f2f3-58f2-4995-ba66-38d83b4c2e41 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 108727d9-7bf4-4042-9a0f-8b07a5a12f91 kind: Event lastTimestamp: "2026-08-05T17:35:28Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:35:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:28Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw.18c8f9454eeae58e namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: b3e21ee8-5101-4930-996c-c68a02530f64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a0ee2-predictor-7f8d8c965f namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: 6bb12419-1780-48d2-8f30-6d77726d8457 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: 'Created pod: error-404-isvc-a0ee2-predictor-7f8d8c965f-kj7sw' metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor-7f8d8c965f.18c8f8c3a3ee8b8a namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: a2d83cca-a332-46c0-aa29-134c84b06e36 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: 68a78229-88b8-42bb-a497-f1c1e169e984 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Scaled up replica set error-404-isvc-a0ee2-predictor-7f8d8c965f from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:11Z" name: error-404-isvc-a0ee2-predictor.18c8f8c3a1c531b1 namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 28fff2b9-bb01-4bef-a360-e96353021b9e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 8e893a02-fa12-41c9-8cca-471aa6e88599 kind: Event lastTimestamp: "2026-08-05T17:26:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:41Z" name: error-404-isvc-a0ee2-predictor.18c8f8c72060b833 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: b07d3674-7938-4b0b-8b77-375c298fe2f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 8e893a02-fa12-41c9-8cca-471aa6e88599 kind: Event lastTimestamp: "2026-08-05T17:26:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:41Z" name: error-404-isvc-a0ee2-predictor.18c8f8c720dd5b5b namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 4698f7f5-eba0-47be-9258-25160a7e0a0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 8e893a02-fa12-41c9-8cca-471aa6e88599 kind: Event lastTimestamp: "2026-08-05T17:27:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:26Z" name: error-404-isvc-a0ee2-predictor.18c8f8ce1ea04e0b namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: f883223c-acad-4a9f-8fba-4f6d944ef6e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 8e893a02-fa12-41c9-8cca-471aa6e88599 kind: Event lastTimestamp: "2026-08-05T17:27:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:26Z" name: error-404-isvc-a0ee2-predictor.18c8f8ce1f143add namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: dda1449d-1627-4b66-b347-e72e0a7ca652 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a0ee2 namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 047cd23a-a0e8-4541-be58-53654296cb69 kind: Event lastTimestamp: "2026-08-05T17:26:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:58Z" name: error-404-isvc-a0ee2.18c8f8c3a214cfe1 namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: 12787fb9-9482-4940-87a0-c52ba70ddc85 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a0ee2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 047cd23a-a0e8-4541-be58-53654296cb69 kind: Event lastTimestamp: "2026-08-05T17:26:58Z" message: InferenceService [error-404-isvc-a0ee2] is Ready metadata: creationTimestamp: "2026-08-05T17:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:58Z" name: error-404-isvc-a0ee2.18c8f8ce95ff7134 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 9f76c72f-0c28-4acc-8fa3-b61db60d2ecd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:27:11.377672Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d9755-predictor-78b5549d57-wh2st to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:27:11Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d19a94ab31 namespace: kserve-ci-e2e-test resourceVersion: "16555" uid: b07e49eb-53d9-40a6-a5f2-22608838eae8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d9755-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d1a4c5bc86 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: b004062a-0f32-44cd-8499-ceb378b6eef6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:27:12Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:27:12Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d1d7d85a87 namespace: kserve-ci-e2e-test resourceVersion: "16603" uid: 84257936-1f62-4aac-9ff1-637f21e8d84b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:27:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:12Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d1d936d46e namespace: kserve-ci-e2e-test resourceVersion: "16605" uid: e7914b2e-38ce-42d9-a85f-dcf2dac7417d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:27:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:12Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d1dc7a342c namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 305199f5-ade8-40bc-ace2-4a5ff391242d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:27:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:12Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d1dce6cc49 namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 4842d84f-ab21-4a57-860b-9625a0f9d5bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:27:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:12Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d1dcefe534 namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: e4cfd3b8-a194-402d-b950-bae8351faa41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:27:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:12Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d1e04540c9 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: efd392a0-e882-4981-81a5-0a87c59cc52a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:27:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:12Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d1e0c81b3f namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 3ecda3e2-fb18-465b-987e-2a5e412d38a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:36:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:29Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f8d21eeabf70 namespace: kserve-ci-e2e-test resourceVersion: "20954" uid: 66ba3506-865c-4f6d-a43d-bf43d7b49b50 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f952c3e239e7 namespace: kserve-ci-e2e-test resourceVersion: "20810" uid: b7c37762-48db-46fc-98c2-0e490f97e2c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f952c3e2b5cd namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 300e16b8-d644-49b6-a014-afc68bbef8bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9755-predictor-78b5549d57-wh2st namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0fe55532-2a2b-41c1-92d4-dc7f0e201ccc kind: Event lastTimestamp: "2026-08-05T17:36:29Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:29Z" name: error-404-isvc-d9755-predictor-78b5549d57-wh2st.18c8f953934b26f4 namespace: kserve-ci-e2e-test resourceVersion: "20953" uid: 4806161b-9e35-49c3-9e65-9c5881245716 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d9755-predictor-78b5549d57 namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: 8b1d35ab-2b1f-436b-976b-18a44a841c78 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: 'Created pod: error-404-isvc-d9755-predictor-78b5549d57-wh2st' metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:11Z" name: error-404-isvc-d9755-predictor-78b5549d57.18c8f8d199c645ca namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 639cfb0c-7290-40b9-9837-37b2d14002f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: 0b1d9c28-bd5e-4d98-9bb8-756819801904 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Scaled up replica set error-404-isvc-d9755-predictor-78b5549d57 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:11Z" name: error-404-isvc-d9755-predictor.18c8f8d1979be4bd namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: afed3b9d-11f6-4c1c-b3fa-b950e25f3221 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: aed198f4-ee56-4230-818c-98aa28b48b41 kind: Event lastTimestamp: "2026-08-05T17:27:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:41Z" name: error-404-isvc-d9755-predictor.18c8f8d515f37b6b namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: e6a9c9b2-452b-4564-9b58-fa59966a1a17 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: aed198f4-ee56-4230-818c-98aa28b48b41 kind: Event lastTimestamp: "2026-08-05T17:27:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:41Z" name: error-404-isvc-d9755-predictor.18c8f8d516330f1b namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 29d1edd8-a6bb-49ed-a47b-770536db294d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:27:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: aed198f4-ee56-4230-818c-98aa28b48b41 kind: Event lastTimestamp: "2026-08-05T17:28:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:28:26Z" name: error-404-isvc-d9755-predictor.18c8f8dc139a5a8a namespace: kserve-ci-e2e-test resourceVersion: "17274" uid: 7d6d3be0-eb92-414c-bf64-345d4d0a1cae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:27:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: aed198f4-ee56-4230-818c-98aa28b48b41 kind: Event lastTimestamp: "2026-08-05T17:28:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:28:26Z" name: error-404-isvc-d9755-predictor.18c8f8dc14050df0 namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: eb3289b7-0b5b-4c20-bcb3-4d1bdf8ad412 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d9755 namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: 054b819e-551b-4b9b-a22b-426dc352409f kind: Event lastTimestamp: "2026-08-05T17:27:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:27:59Z" name: error-404-isvc-d9755.18c8f8d1979b8438 namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: 5a254430-40bd-435d-bd88-81c48fcf7f76 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d9755 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: 054b819e-551b-4b9b-a22b-426dc352409f kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-d9755": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d9755": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:27:11Z" name: error-404-isvc-d9755.18c8f8d1a0bd126a namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: c6e5ecf2-74d3-478d-a38b-1ef7b114ae6d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d9755 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: 054b819e-551b-4b9b-a22b-426dc352409f kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d9755": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:27:11Z" name: error-404-isvc-d9755.18c8f8d1a0bd6b2b namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: 9b577871-aea1-47bf-851c-913665384db4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d9755 namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 054b819e-551b-4b9b-a22b-426dc352409f kind: Event lastTimestamp: "2026-08-05T17:27:59Z" message: InferenceService [error-404-isvc-d9755] is Ready metadata: creationTimestamp: "2026-08-05T17:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:27:59Z" name: error-404-isvc-d9755.18c8f8dcda838903 namespace: kserve-ci-e2e-test resourceVersion: "17031" uid: 08a599aa-9fa7-45b7-9e3a-c46b22d35657 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:47:01.774334Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26676" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:47:01Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e6c3c0103b namespace: kserve-ci-e2e-test resourceVersion: "26680" uid: 94127de2-fdd8-41cf-852c-495d231c6526 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-db0ba-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e6d2d258aa namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: 5fef9607-b977-4345-a1b1-e0db1f44054b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26685" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:47:02Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e7019a5087 namespace: kserve-ci-e2e-test resourceVersion: "26710" uid: 5cc2a67b-1b42-47cd-b5cb-b466edf7ab37 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e702ed6604 namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: c4e35334-5fbf-4569-8dab-4214d777da57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e70671b765 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 8f775822-f272-4423-9902-c31d0cf94930 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e706df2bf0 namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: 5c0d3626-9501-4eb4-bb4c-67f061ec1643 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e706e88d3e namespace: kserve-ci-e2e-test resourceVersion: "26717" uid: acd95f80-98e0-4703-9504-03de9da9cea2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e709ab653b namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 3e02a140-7183-487e-9808-3905504b5a31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e70a2a4b27 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 8f8710fa-0799-4bac-afc7-1f58d8ba9083 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:47:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:47:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:39Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8f9e7355e3672 namespace: kserve-ci-e2e-test resourceVersion: "27051" uid: b4ae6092-ab1f-4d5a-bef2-33fb39cfd013 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:56:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:56:16Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8fa67eba69169 namespace: kserve-ci-e2e-test resourceVersion: "30807" uid: 1f957395-94b0-451d-b6e6-55f83a2133b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 8b670052-3f8a-4f70-82a8-d772ab208ea0 kind: Event lastTimestamp: "2026-08-05T17:56:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:56:16Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp.18c8fa67eba7c67d namespace: kserve-ci-e2e-test resourceVersion: "30808" uid: bf66b5ac-b586-4c85-85d7-0d7b48dada60 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-db0ba-predictor-68c79c6fb7 namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: 519d42ef-0d0e-4f07-b0e0-036df2a0800a kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: 'Created pod: error-404-isvc-db0ba-predictor-68c79c6fb7-mpttp' metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:01Z" name: error-404-isvc-db0ba-predictor-68c79c6fb7.18c8f9e6c32bf3aa namespace: kserve-ci-e2e-test resourceVersion: "26678" uid: 716e6aab-2280-4c60-893b-37b43e5f5a9e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: cb2c4e2d-0ec3-46bc-9868-2151e7549a4c kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: Scaled up replica set error-404-isvc-db0ba-predictor-68c79c6fb7 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:01Z" name: error-404-isvc-db0ba-predictor.18c8f9e6c09cb96b namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: 444663b5-9721-407e-8214-4e6bb37dec4a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:47:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: a13fe687-7680-4404-93fe-fb13844c0cb9 kind: Event lastTimestamp: "2026-08-05T17:47:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:46Z" name: error-404-isvc-db0ba-predictor.18c8f9ea3f39b3a8 namespace: kserve-ci-e2e-test resourceVersion: "27112" uid: d7ff4816-865b-4f8f-ac70-3ba0be5c5df2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:47:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: a13fe687-7680-4404-93fe-fb13844c0cb9 kind: Event lastTimestamp: "2026-08-05T17:47:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:46Z" name: error-404-isvc-db0ba-predictor.18c8f9ea3faac215 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 6af207b4-14cd-4ba3-91dc-461ca876b568 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: a13fe687-7680-4404-93fe-fb13844c0cb9 kind: Event lastTimestamp: "2026-08-05T17:48:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:16Z" name: error-404-isvc-db0ba-predictor.18c8f9edbe7576c2 namespace: kserve-ci-e2e-test resourceVersion: "27449" uid: 7874dcde-153a-437e-b573-39e949be3812 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: a13fe687-7680-4404-93fe-fb13844c0cb9 kind: Event lastTimestamp: "2026-08-05T17:48:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:16Z" name: error-404-isvc-db0ba-predictor.18c8f9edbef01798 namespace: kserve-ci-e2e-test resourceVersion: "27450" uid: 18b99880-8a68-41d7-96b0-e9ef8571aed6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db0ba namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 07c63a84-68a5-4f0e-ad82-748f3b5640b8 kind: Event lastTimestamp: "2026-08-05T17:47:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:49Z" name: error-404-isvc-db0ba.18c8f9e6c0c10687 namespace: kserve-ci-e2e-test resourceVersion: "27160" uid: 83924ff0-531d-41b8-bb69-64eb44467ad4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db0ba namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 07c63a84-68a5-4f0e-ad82-748f3b5640b8 kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-db0ba": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-db0ba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:01Z" name: error-404-isvc-db0ba.18c8f9e6c17eb2f1 namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 3667988f-0faa-41d2-8af7-a95b27c5a1cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db0ba namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 07c63a84-68a5-4f0e-ad82-748f3b5640b8 kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-db0ba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:01Z" name: error-404-isvc-db0ba.18c8f9e6c17f04e2 namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: 599c1df4-ada4-4cb5-8ae0-1a478ce18467 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db0ba namespace: kserve-ci-e2e-test resourceVersion: "27157" uid: 07c63a84-68a5-4f0e-ad82-748f3b5640b8 kind: Event lastTimestamp: "2026-08-05T17:47:49Z" message: InferenceService [error-404-isvc-db0ba] is Ready metadata: creationTimestamp: "2026-08-05T17:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:49Z" name: error-404-isvc-db0ba.18c8f9f1f4a0b873 namespace: kserve-ci-e2e-test resourceVersion: "27158" uid: 88582050-4adc-4f1e-8162-11da8fcaa38c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:37:36.611254Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21836" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:37:36Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9632d67e0b4 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 9945b415-e4b3-44b1-8ae0-e7e2503dcece reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e149b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:36Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f963375e8d59 namespace: kserve-ci-e2e-test resourceVersion: "21859" uid: f9651479-7714-4b84-9b09-9e19ad9150b0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21848" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:37:37Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9636b99bdb4 namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 90627020-3697-4cab-8c0b-a1fcd654359a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9636d11c4a8 namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: ddf482c1-8b80-44b3-b9fc-2933b23bddf5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9636fd7de59 namespace: kserve-ci-e2e-test resourceVersion: "21894" uid: 2ae2fb90-335f-4e52-b7ab-5806c2d629c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9637043755b namespace: kserve-ci-e2e-test resourceVersion: "21895" uid: ce935163-f361-4073-9c39-3ded93da9033 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f963704c3faf namespace: kserve-ci-e2e-test resourceVersion: "21896" uid: f9a841ff-3a5a-4295-bc32-75591fd61473 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f963738a37be namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 05126b29-dc62-4917-963b-a37377237515 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9637413bb93 namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: 3c2d5d66-fc8a-4b00-be6f-414a72498490 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-05T17:37:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:46:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:54Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9638f5e8534 namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 824283d2-3583-4066-b2d6-9479abbb670e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9e4535004cb namespace: kserve-ci-e2e-test resourceVersion: "26382" uid: a647ffd6-34c9-4c03-9ee4-fe9da5dfe203 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9e453510dad namespace: kserve-ci-e2e-test resourceVersion: "26385" uid: 3ed7d7ab-f685-4e68-9e20-080a2fc9b456 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: b528315e-f99c-4518-bdf1-87a2863c69a6 kind: Event lastTimestamp: "2026-08-05T17:46:54Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:54Z" name: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf.18c8f9e503d03cd5 namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 52d8ce4b-8060-4739-9ffd-0aebcac37185 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e149b-predictor-6cc79c7d67 namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 26fc61e9-f275-4cd9-93ac-0a243ab4d13f kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'Created pod: error-404-isvc-e149b-predictor-6cc79c7d67-7r5pf' metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:36Z" name: error-404-isvc-e149b-predictor-6cc79c7d67.18c8f9632d006212 namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: d070df64-7bb7-43ba-a47d-949519b3fd4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: 3dfc76d8-5ce2-4101-bd11-48254437bd2a kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Scaled up replica set error-404-isvc-e149b-predictor-6cc79c7d67 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:36Z" name: error-404-isvc-e149b-predictor.18c8f9632b1eb826 namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: 18ad1899-fbab-41a5-be6c-f9dd6b965549 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: fe91570c-1c68-4018-9e7f-d289a62e3b15 kind: Event lastTimestamp: "2026-08-05T17:38:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:06Z" name: error-404-isvc-e149b-predictor.18c8f966a9cbaf5b namespace: kserve-ci-e2e-test resourceVersion: "22148" uid: 1aee999c-d7d4-41e5-854e-5cc1d13915b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: fe91570c-1c68-4018-9e7f-d289a62e3b15 kind: Event lastTimestamp: "2026-08-05T17:38:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:06Z" name: error-404-isvc-e149b-predictor.18c8f966aa4d27f2 namespace: kserve-ci-e2e-test resourceVersion: "22149" uid: a1e381dd-1c19-49ac-8971-0ed02e0bf350 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:38:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: fe91570c-1c68-4018-9e7f-d289a62e3b15 kind: Event lastTimestamp: "2026-08-05T17:38:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:51Z" name: error-404-isvc-e149b-predictor.18c8f96da80d0cfb namespace: kserve-ci-e2e-test resourceVersion: "22531" uid: 5ac18040-0dcb-4fb6-988c-970a2150f09a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:38:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: fe91570c-1c68-4018-9e7f-d289a62e3b15 kind: Event lastTimestamp: "2026-08-05T17:38:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:51Z" name: error-404-isvc-e149b-predictor.18c8f96da87afa05 namespace: kserve-ci-e2e-test resourceVersion: "22532" uid: 6e5f1a4c-3c22-4ca8-9188-8933e4992620 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e149b namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: ac1d046d-13c9-45dd-a50f-88379c428583 kind: Event lastTimestamp: "2026-08-05T17:38:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:38:24Z" name: error-404-isvc-e149b.18c8f9632b437d00 namespace: kserve-ci-e2e-test resourceVersion: "22286" uid: 813249a8-bda3-4fcf-8ad9-5b5a503ea6dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e149b namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: ac1d046d-13c9-45dd-a50f-88379c428583 kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-e149b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e149b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:36Z" name: error-404-isvc-e149b.18c8f9632bc47e02 namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: e5e689c5-4710-4978-997e-f5c812187dff reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e149b namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: ac1d046d-13c9-45dd-a50f-88379c428583 kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e149b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:36Z" name: error-404-isvc-e149b.18c8f9632bc4c4ea namespace: kserve-ci-e2e-test resourceVersion: "21831" uid: 4b2687d0-d910-4bfb-b062-2af598613bb8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e149b namespace: kserve-ci-e2e-test resourceVersion: "22284" uid: ac1d046d-13c9-45dd-a50f-88379c428583 kind: Event lastTimestamp: "2026-08-05T17:38:24Z" message: InferenceService [error-404-isvc-e149b] is Ready metadata: creationTimestamp: "2026-08-05T17:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:38:24Z" name: error-404-isvc-e149b.18c8f96e4b9757e3 namespace: kserve-ci-e2e-test resourceVersion: "22285" uid: f8154486-e3c6-4667-9a6f-0082f398b500 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:35:26.307083Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20232" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fba55-predictor-676475b57f-fljvk to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:35:26Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f944d6ac085d namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: 50c317d1-0361-40a4-8e0b-b825fa3487f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fba55-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f944e2b126d3 namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: 466fce83-e1d3-41b6-ab4b-c01b00dbb4c6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:35:27Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:35:27Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f94513fe9fb5 namespace: kserve-ci-e2e-test resourceVersion: "20272" uid: d6110373-47ae-422e-aa42-703f4138906f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:35:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:27Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f945157d285a namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: c3912b7f-1690-4eb2-a1c1-df13f47bef77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:35:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:27Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f945189e4707 namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: 48507d7b-79f4-4e53-aaf3-a1df11acd2c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:35:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:27Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f945190f4bd8 namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: 61e61480-f637-4fb4-a0fc-bbe05edcbae2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:35:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:27Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f945191ab8ad namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: f7706256-0308-444e-b845-131625efd1cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:35:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:27Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f9451bee0622 namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 3f9b81e7-a893-4bb7-a345-96296c23c7b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:35:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:27Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f9451c617f5d namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 3cdf26ac-2dd1-4a96-a34f-35fbe68ba70a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:36:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:03Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f945265016f6 namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 39e6da62-51e9-44a5-bed2-0c24dda555fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:36Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f95531598ae6 namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: de584128-95d2-4b03-83c6-af114f6cbcc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:36Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f955315ad774 namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: ca2ad22e-34c3-449b-931c-b2db156a4457 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fba55-predictor-676475b57f-fljvk namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 366121e0-36b3-4704-a224-ee0e872f94e4 kind: Event lastTimestamp: "2026-08-05T17:36:38Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:38Z" name: error-404-isvc-fba55-predictor-676475b57f-fljvk.18c8f955ae8ac1c0 namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: a84c7651-e55f-427c-9891-4d39abf3de23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fba55-predictor-676475b57f namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: beb83e00-f337-4e74-9ecf-d3ac03860c18 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: 'Created pod: error-404-isvc-fba55-predictor-676475b57f-fljvk' metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:35:26Z" name: error-404-isvc-fba55-predictor-676475b57f.18c8f944d63e34e9 namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: 4ca371a4-8122-41be-800c-ccfd7c4d3e2b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: 3a9633da-c4d4-4ed6-b218-db53ab2ca7ca kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Scaled up replica set error-404-isvc-fba55-predictor-676475b57f from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:35:26Z" name: error-404-isvc-fba55-predictor.18c8f944d3ffc721 namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 0b003b03-b4e9-4453-82c4-833a9db3f3b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:35:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: d9c45f84-3e00-4368-b1a0-f5d61b57dbec kind: Event lastTimestamp: "2026-08-05T17:35:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:35:56Z" name: error-404-isvc-fba55-predictor.18c8f948527b1ae6 namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 3942d001-8508-4f98-a03c-cb6c83ae5959 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:35:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: d9c45f84-3e00-4368-b1a0-f5d61b57dbec kind: Event lastTimestamp: "2026-08-05T17:35:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:35:56Z" name: error-404-isvc-fba55-predictor.18c8f94852f6ad93 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: da74a877-455b-4884-bea0-4dc3f900e18b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: d9c45f84-3e00-4368-b1a0-f5d61b57dbec kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-fba55-predictor.18c8f94f50833007 namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: 72fa1d4a-ca53-4d3e-8461-890e99533d0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: d9c45f84-3e00-4368-b1a0-f5d61b57dbec kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: error-404-isvc-fba55-predictor.18c8f94f50f7618f namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 1754fd88-2a50-492e-b966-09fc3109a543 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fba55 namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: e958d08f-d8e1-497b-89da-d0885618df63 kind: Event lastTimestamp: "2026-08-05T17:36:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:13Z" name: error-404-isvc-fba55.18c8f944d41e07a6 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: a562ac77-cb40-4048-8beb-80da0ee2884a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fba55 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: e958d08f-d8e1-497b-89da-d0885618df63 kind: Event lastTimestamp: "2026-08-05T17:36:13Z" message: InferenceService [error-404-isvc-fba55] is Ready metadata: creationTimestamp: "2026-08-05T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:13Z" name: error-404-isvc-fba55.18c8f94fe2316149 namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: a8277067-1174-49c9-967b-5eff6c60aeed reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:24:40.971280Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-08-05T17:24:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:24:40Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8ae95a92985 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: 12f3ab7c-a8bc-41b2-8c11-c92328f3bf14 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14719" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:24:41Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8aeaf93e668 namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 91903d02-b611-4f25-9931-03134ffe3101 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:925fa2b887b6743b69728677063d64dcececcf1143340982442b2ec3dc29490e" already present on machine metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:41Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8aeb11fb201 namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 15dafe3b-5d54-427a-8042-85e315cc34b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:41Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8aeb5535a7c namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: a4a11955-f6a0-4476-a5f9-9554200d1099 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:41Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8aeb5ca6fcd namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 5c57b9f2-f003-4bae-bc99-6537be52752c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:24:46Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1848" metadata: creationTimestamp: "2026-08-05T17:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:46Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8afc75d6c51 namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: abf67ab5-bb14-4985-b073-33e826acd7b5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1848" in 27.204s (27.204s including waiting). Image size: 1555738027 bytes.' metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8b61ce19f9b namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 8c578f0f-5a8b-4f9d-b26c-02c95ef3f367 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8b621fc61a0 namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: de796bcc-9c5f-428c-baaa-2c8025a3aa31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8b623102631 namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: a0e67b90-59fc-4ecc-9638-a6920a673017 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8b6231b8377 namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 5de1062c-2f78-4eab-8475-7917132e3c73 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:25:16Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.005s (3.005s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-08-05T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:16Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8b6d640b24f namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 499884c6-2c01-43d4-a207-2888caf8a81e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:25:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:16Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8b6da571ac3 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: bdf4edf2-d5c5-412a-91ec-7edf1e07d597 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:25:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:16Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8b6dab71b53 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: 8ca6b344-0384-4345-8ee7-767f4d2fe32f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-08-05T17:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8b70c017bcb namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: ec19c6ca-448e-4663-a97c-a1041fd168a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8d1917a6916 namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 2a9657ff-671c-416a-b311-58d6d6a904bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8d1917ac820 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 5e85414f-d498-47ae-bf98-36c9c002ce7c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f6659c94-e049-40d7-a9a2-fbd59847eff6 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc.18c8f8d20ea6814c namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: 477f0d30-9f14-41b7-9728-0e7ca310a29c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-69ddc59866 namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: 5672d06d-abf1-4261-8a05-8615919e9ae0 kind: Event lastTimestamp: "2026-08-05T17:24:40Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-69ddc59866-6f2nc' metadata: creationTimestamp: "2026-08-05T17:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:40Z" name: isvc-sklearn-graph-1-predictor-69ddc59866.18c8f8ae9551f013 namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: b52ecca3-e654-4b7f-9ddb-08d56c7d5c0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14694" uid: 0553b38a-e8e2-4140-a126-46535be9b735 kind: Event lastTimestamp: "2026-08-05T17:24:40Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-69ddc59866 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:40Z" name: isvc-sklearn-graph-1-predictor.18c8f8ae8dc0d866 namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: bccb9a34-1347-42be-8b74-4355c00e7151 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:24:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 3fc5e14c-0bee-4fed-9507-bede6b8cc17e kind: Event lastTimestamp: "2026-08-05T17:25:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:41Z" name: isvc-sklearn-graph-1-predictor.18c8f8b2134b1fee namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: cb1fd4a6-62cd-4be1-ae3d-662901af19d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:24:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 3fc5e14c-0bee-4fed-9507-bede6b8cc17e kind: Event lastTimestamp: "2026-08-05T17:25:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:41Z" name: isvc-sklearn-graph-1-predictor.18c8f8b213c2015c namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 92ad7d4d-8279-45ec-82b8-26521fe4ed08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:25:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 3fc5e14c-0bee-4fed-9507-bede6b8cc17e kind: Event lastTimestamp: "2026-08-05T17:26:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:56Z" name: isvc-sklearn-graph-1-predictor.18c8f8c010419bbe namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: d18c8ced-794c-48a4-b6d2-5e9b3e5e0dd8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:25:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 3fc5e14c-0bee-4fed-9507-bede6b8cc17e kind: Event lastTimestamp: "2026-08-05T17:26:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:56Z" name: isvc-sklearn-graph-1-predictor.18c8f8c010ad73b0 namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: 6496137a-d2eb-47fb-9c2a-e25f848d9c13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-08-05T17:24:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 73e7fa56-0b90-4b5c-b7e0-3430e4b40252 kind: Event lastTimestamp: "2026-08-05T17:26:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:33Z" name: isvc-sklearn-graph-1.18c8f8ae93dc0223 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: 89fbed28-1866-4314-9f97-211ff9122880 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: 73e7fa56-0b90-4b5c-b7e0-3430e4b40252 kind: Event lastTimestamp: "2026-08-05T17:26:33Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-08-05T17:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:33Z" name: isvc-sklearn-graph-1.18c8f8c8c489ff36 namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 11fa95fc-c3a8-47bc-9c69-6a6754f7119e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:24:41.566618Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-756656f574-kg87c to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:24:41Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8aeb92547fe namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 81b9aa29-10c4-489a-b11d-8a1d2651375f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:41Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8aec60369f4 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: 077626ef-dedf-4af2-ad31-779d32d075b8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:24:42Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:24:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:24:42Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8aef700d65a namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: 89fa8520-f4d8-4525-a093-154233259534 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:24:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:925fa2b887b6743b69728677063d64dcececcf1143340982442b2ec3dc29490e" already present on machine metadata: creationTimestamp: "2026-08-05T17:24:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:42Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8aef86c59c6 namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: ce5183d9-eb96-48a8-89fa-94d725e65023 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:24:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-08-05T17:24:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:42Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8aefb4a7009 namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: e028d241-01c6-4925-9931-84215eeca14d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:24:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-08-05T17:24:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:42Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8aefbbcabff namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: b67d299b-238a-43f9-aab3-1e507a3a892d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:24:47Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1848" metadata: creationTimestamp: "2026-08-05T17:24:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:47Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8b0048d0c91 namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: 09fa634c-cb09-4536-9dc5-b17446da3470 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1848" in 26.247s (26.247s including waiting). Image size: 1555738027 bytes.' metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8b62101ef80 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: a7884b4a-9f7f-421c-bf30-10c7f85a33ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8b6260141c5 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: fc47e62b-fdc1-4ebc-a51a-7cf8a4b60087 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8b62709d8c2 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 1afa613d-5563-47c8-b64f-f7bcf0740b88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8b62715a0ce namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: bb4245b4-b9b9-4a0b-b41b-cbb144750606 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:25:16Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.951s (2.951s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-08-05T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:16Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8b6d7078477 namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 11e43a93-cdaa-4652-8864-0ef04439259c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:25:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:16Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8b6da61fed4 namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 7c4e7bd7-e400-4264-ad3a-6e21d3a04dfd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:25:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:16Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8b6dabf1485 namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: cad63c4c-1b7e-436b-94f8-9b4109e1b5a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-08-05T17:25:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:27:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:25:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:14Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8b74829bd53 namespace: kserve-ci-e2e-test resourceVersion: "16650" uid: 50304eaa-cd6f-4796-abd4-d3569cefdbc9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8d18da8fab4 namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: a2ac08bf-e428-4c9b-bde6-bc538f6e1ca7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8d18da972bb namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 5faaa3e5-3acc-49d7-9982-01b18a71f22a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-756656f574-kg87c namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 74e14104-9c4f-456c-966a-68b55168dc02 kind: Event lastTimestamp: "2026-08-05T17:27:14Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:27:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:14Z" name: isvc-sklearn-graph-2-predictor-756656f574-kg87c.18c8f8d24a81e6a4 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 59d07eed-3d70-4c84-8b0f-9948198e6037 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-756656f574 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 5df169ed-696a-49ef-adfc-66f5a3b09440 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-756656f574-kg87c' metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:41Z" name: isvc-sklearn-graph-2-predictor-756656f574.18c8f8aeb8cdde2e namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 7ece0b82-50d5-457a-81eb-dce9a4896072 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 7847df7a-3737-467f-91c8-491ddbd820a2 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-756656f574 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:41Z" name: isvc-sklearn-graph-2-predictor.18c8f8aeb2dbe8b1 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 41af2157-1bd2-4d2f-b7ff-074f68d0a1b6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: b5613b28-227e-403e-87fb-d019198f07b6 kind: Event lastTimestamp: "2026-08-05T17:25:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:56Z" name: isvc-sklearn-graph-2-predictor.18c8f8b2313a13ef namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 25ee33f0-0587-4925-82ce-adb0a17fb1d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: b5613b28-227e-403e-87fb-d019198f07b6 kind: Event lastTimestamp: "2026-08-05T17:25:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:56Z" name: isvc-sklearn-graph-2-predictor.18c8f8b231b36322 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: aa7e0e78-0af3-4b12-bd42-9506ae68cb12 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:25:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: b5613b28-227e-403e-87fb-d019198f07b6 kind: Event lastTimestamp: "2026-08-05T17:26:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:56Z" name: isvc-sklearn-graph-2-predictor.18c8f8bcae057bc7 namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 2a8e1cea-e571-437f-87c4-c4e9b2c9b2b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:25:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: b5613b28-227e-403e-87fb-d019198f07b6 kind: Event lastTimestamp: "2026-08-05T17:26:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:56Z" name: isvc-sklearn-graph-2-predictor.18c8f8bcae74f407 namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: b2e546ad-9eea-4e70-bb3f-d64ec9828d19 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 4a0e03b9-dccf-4499-b7b0-dcce38f90661 kind: Event lastTimestamp: "2026-08-05T17:26:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:34Z" name: isvc-sklearn-graph-2.18c8f8aeb2ddbc54 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: fd9de6ef-9f43-427c-b884-cbab7c2f2cf0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 4a0e03b9-dccf-4499-b7b0-dcce38f90661 kind: Event lastTimestamp: "2026-08-05T17:26:34Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-08-05T17:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:34Z" name: isvc-sklearn-graph-2.18c8f8c8ff34e36c namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 12719380-5565-420e-ae48-02b9290da5fe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:24:41.274994Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-dxz4x to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:24:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8aea7c374b0 namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: a743d11d-8aed-46ed-8280-9be299ca0e5d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:24:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8aec35ee15b namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: 48cba5e7-ed1e-49df-bb72-7ca546f4721c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:925fa2b887b6743b69728677063d64dcececcf1143340982442b2ec3dc29490e" already present on machine metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8aec4c12b83 namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 903a7ab0-48ac-4e13-9da5-adc8c26b4bc0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8aec8cff4a1 namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: 1a013081-c617-4ee2-a4f9-ddcd0c1cf4d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8aec94d22f6 namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: e12b95ce-5ef6-472a-8997-92d42c660807 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:24:46Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-08-05T17:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:46Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8afc7c97fb6 namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 54e3c5ef-432e-4ab0-b4c4-0e6befab7363 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.623s (27.623s including waiting). Image size: 1307355899 bytes.' metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8b636404bc2 namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: d868e133-de77-4de3-9b77-9dece5b50977 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8b63b53a10a namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: 7c2d7d68-ca8d-48bb-9bb9-e66678686bed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8b63c46c3f7 namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 26d25384-a188-43d2-b23a-d0078f624150 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:25:13Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-08-05T17:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8b63c536d2c namespace: kserve-ci-e2e-test resourceVersion: "15184" uid: 95633c54-d6a8-46ab-9c84-d8218b15f228 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:25:16Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.565s (2.565s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-08-05T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8b6d5452f13 namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: 51a241d0-36f7-4f2f-af31-eedbe4bc2410 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:25:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8b6da01da2e namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 0a56faf4-208c-47bd-ae90-e92ecfa14320 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:25:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8b6da694283 namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 8b24f9c6-63cb-4c12-8d14-41c49ca31255 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-08-05T17:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8b70c3c699e namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: d78456d3-55eb-4e2d-861b-b95f007c0675 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8d195663e5c namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: 04ad4e19-9aa1-4fa2-910e-91ee19b9cc61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8d1956697ee namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: 2ec66d75-9813-47ae-be26-ec9f8bce8d7c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 618885d9-612d-4e62-8742-7ce9bf9e7224 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-dxz4x.18c8f8d20ea7b9e7 namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: 1b26e981-8c99-4bd6-9b0f-eda21ad955a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: e29e41b9-00ac-4ae0-bb92-0ccfdbb482c2 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-dxz4x' metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:41Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c8f8aea7565c04 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: ff9b961c-34d4-4b16-8b73-84a19e675727 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: a07ac4c1-28ad-4b60-8e51-a075e63323f5 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:41Z" name: isvc-xgboost-graph-predictor.18c8f8aea40d1011 namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 6a536019-9344-4dae-9a26-77cc91a2b92e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: f8fce2c8-8403-4216-8144-34227578827c kind: Event lastTimestamp: "2026-08-05T17:25:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:56Z" name: isvc-xgboost-graph-predictor.18c8f8b222a00b24 namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: 06510f38-08fb-47c7-8554-c670b13b7a68 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: f8fce2c8-8403-4216-8144-34227578827c kind: Event lastTimestamp: "2026-08-05T17:25:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:56Z" name: isvc-xgboost-graph-predictor.18c8f8b22310ed89 namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: a785593e-a025-4ea9-bc24-01d2b6b37c58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:25:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: f8fce2c8-8403-4216-8144-34227578827c kind: Event lastTimestamp: "2026-08-05T17:26:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:41Z" name: isvc-xgboost-graph-predictor.18c8f8bc9f491a20 namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: 380338dc-688b-4507-8460-c94ee8c26930 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:25:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: f8fce2c8-8403-4216-8144-34227578827c kind: Event lastTimestamp: "2026-08-05T17:26:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:41Z" name: isvc-xgboost-graph-predictor.18c8f8bc9fbd8803 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 1e6260c3-afff-49c3-b6b9-1040a7826fe7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14745" uid: 7a06e43a-0a65-4bab-87fe-ddf919c41355 kind: Event lastTimestamp: "2026-08-05T17:26:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:13Z" name: isvc-xgboost-graph.18c8f8aea4714e11 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 57e424da-b6f3-4e19-8599-44c4b6536c3b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: 7a06e43a-0a65-4bab-87fe-ddf919c41355 kind: Event lastTimestamp: "2026-08-05T17:26:13Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-08-05T17:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:13Z" name: isvc-xgboost-graph.18c8f8c41c4b853d namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: 65749a34-2abc-4656-9867-5b3a91f1dfef reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:27:00.920112Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-d84c755c4-7922z namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: ec59023a-b86f-4b3a-9bcb-f4931640a3a7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-d84c755c4-7922z to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:27:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:27:00Z" name: model-chainer-d84c755c4-7922z.18c8f8cf2b42fc74 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: fc80ff48-0064-4ea3-9973-6bacc192db7e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-d84c755c4-7922z namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: ec59023a-b86f-4b3a-9bcb-f4931640a3a7 kind: Event lastTimestamp: "2026-08-05T17:27:01Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:27:01Z" name: model-chainer-d84c755c4-7922z.18c8f8cf44d9bb41 namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 5e4a9a87-2e96-4ab7-a89a-b20185f9907e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d84c755c4-7922z namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: ec59023a-b86f-4b3a-9bcb-f4931640a3a7 kind: Event lastTimestamp: "2026-08-05T17:27:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:01Z" name: model-chainer-d84c755c4-7922z.18c8f8cf4638e9d9 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 650ac205-c05b-4d42-b700-76181d471de4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d84c755c4-7922z namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: ec59023a-b86f-4b3a-9bcb-f4931640a3a7 kind: Event lastTimestamp: "2026-08-05T17:27:01Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-08-05T17:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:01Z" name: model-chainer-d84c755c4-7922z.18c8f8cf49394579 namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: d114ce60-3ffc-4f16-9382-26ed9891a4ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d84c755c4-7922z namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: ec59023a-b86f-4b3a-9bcb-f4931640a3a7 kind: Event lastTimestamp: "2026-08-05T17:27:01Z" message: Started container model-chainer metadata: creationTimestamp: "2026-08-05T17:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:01Z" name: model-chainer-d84c755c4-7922z.18c8f8cf49a78ba2 namespace: kserve-ci-e2e-test resourceVersion: "16345" uid: 4e0282de-3b88-44de-9bac-c0d637de7420 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d84c755c4-7922z namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: ec59023a-b86f-4b3a-9bcb-f4931640a3a7 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: model-chainer-d84c755c4-7922z.18c8f8d1855bc357 namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: 50312e67-31e2-480a-8600-aa4b8635be37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d84c755c4-7922z namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: ec59023a-b86f-4b3a-9bcb-f4931640a3a7 kind: Event lastTimestamp: "2026-08-05T17:27:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:37Z" name: model-chainer-d84c755c4-7922z.18c8f8d1ea18553a namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: 8aeda206-f769-40c7-805e-d2f2353100fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-d84c755c4 namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: ae2435b7-1281-4786-bc6d-0a67266f2fe7 kind: Event lastTimestamp: "2026-08-05T17:27:00Z" message: 'Created pod: model-chainer-d84c755c4-7922z' metadata: creationTimestamp: "2026-08-05T17:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:00Z" name: model-chainer-d84c755c4.18c8f8cf2ad12e9c namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 3755bbd0-2bc4-4168-8d95-71674a7a08d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 4ece2011-f993-48ff-af20-4ed1d568f877 kind: Event lastTimestamp: "2026-08-05T17:27:00Z" message: Scaled up replica set model-chainer-d84c755c4 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:00Z" name: model-chainer.18c8f8cf29cc98fb namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 3e7829fd-6224-4d43-b06a-d161c6bdcc6b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 05878351-c4d0-4206-819d-736d44c284f1 kind: Event lastTimestamp: "2026-08-05T17:27:00Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-08-05T17:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:27:00Z" name: model-chainer.18c8f8cf2f9c3eb5 namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 54de3b99-588b-4430-ab55-a1f0145c50fa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:37:26.179578Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-27640-6ff445f5cd-98hvc namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: 34319feb-40bd-45aa-a2e1-13a9dcf8742c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-27640-6ff445f5cd-98hvc to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:37:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:37:26Z" name: sequence-graph-27640-6ff445f5cd-98hvc.18c8f960bfa12b73 namespace: kserve-ci-e2e-test resourceVersion: "21607" uid: 73a633e5-9e94-4ebf-94fd-82db215f337c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-27640-6ff445f5cd-98hvc namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 34319feb-40bd-45aa-a2e1-13a9dcf8742c kind: Event lastTimestamp: "2026-08-05T17:37:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-27640-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:26Z" name: sequence-graph-27640-6ff445f5cd-98hvc.18c8f960c95eaaf8 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: 34439244-5ed3-4b32-a02d-9c05b2106b0d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-27640-6ff445f5cd-98hvc namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 34319feb-40bd-45aa-a2e1-13a9dcf8742c kind: Event lastTimestamp: "2026-08-05T17:37:27Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:37:27Z" name: sequence-graph-27640-6ff445f5cd-98hvc.18c8f960fcfd88d5 namespace: kserve-ci-e2e-test resourceVersion: "21629" uid: a94dca8b-a660-48f7-955c-5bf48fc67d63 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-27640} kind: Pod name: sequence-graph-27640-6ff445f5cd-98hvc namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 34319feb-40bd-45aa-a2e1-13a9dcf8742c kind: Event lastTimestamp: "2026-08-05T17:37:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:27Z" name: sequence-graph-27640-6ff445f5cd-98hvc.18c8f960fe9b94a1 namespace: kserve-ci-e2e-test resourceVersion: "21631" uid: e9d8eebe-9b1d-4838-8260-490d5d236950 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-27640} kind: Pod name: sequence-graph-27640-6ff445f5cd-98hvc namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 34319feb-40bd-45aa-a2e1-13a9dcf8742c kind: Event lastTimestamp: "2026-08-05T17:37:27Z" message: 'Created container: sequence-graph-27640' metadata: creationTimestamp: "2026-08-05T17:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:27Z" name: sequence-graph-27640-6ff445f5cd-98hvc.18c8f96101d82eb3 namespace: kserve-ci-e2e-test resourceVersion: "21632" uid: 14d3e851-df3a-4228-8b37-da26186062f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-27640} kind: Pod name: sequence-graph-27640-6ff445f5cd-98hvc namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 34319feb-40bd-45aa-a2e1-13a9dcf8742c kind: Event lastTimestamp: "2026-08-05T17:37:27Z" message: Started container sequence-graph-27640 metadata: creationTimestamp: "2026-08-05T17:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:27Z" name: sequence-graph-27640-6ff445f5cd-98hvc.18c8f96102493f9c namespace: kserve-ci-e2e-test resourceVersion: "21633" uid: ba97629e-7718-4a84-9f1a-1b02d37495ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-27640} kind: Pod name: sequence-graph-27640-6ff445f5cd-98hvc namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 34319feb-40bd-45aa-a2e1-13a9dcf8742c kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Stopping container sequence-graph-27640 metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:36Z" name: sequence-graph-27640-6ff445f5cd-98hvc.18c8f963198bfebb namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: fe1c8adf-04b7-4f83-bb54-4edab27aa9a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-27640} kind: Pod name: sequence-graph-27640-6ff445f5cd-98hvc namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 34319feb-40bd-45aa-a2e1-13a9dcf8742c kind: Event lastTimestamp: "2026-08-05T17:38:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:38:04Z" name: sequence-graph-27640-6ff445f5cd-98hvc.18c8f963cb3160ef namespace: kserve-ci-e2e-test resourceVersion: "22124" uid: 80ea9aaf-c266-4b9a-bc09-fc7a6b3c1424 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-27640-6ff445f5cd namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 69137e79-bf2c-4837-85c1-9a551843cfe8 kind: Event lastTimestamp: "2026-08-05T17:37:26Z" message: 'Created pod: sequence-graph-27640-6ff445f5cd-98hvc' metadata: creationTimestamp: "2026-08-05T17:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:26Z" name: sequence-graph-27640-6ff445f5cd.18c8f960bf1d61ff namespace: kserve-ci-e2e-test resourceVersion: "21605" uid: 04d5349e-aae8-4a2c-bf50-f413ac3a34fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-27640 namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: 43b2ea09-1e8b-4bbd-8e05-ea3cc60a43c5 kind: Event lastTimestamp: "2026-08-05T17:37:26Z" message: Scaled up replica set sequence-graph-27640-6ff445f5cd from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:26Z" name: sequence-graph-27640.18c8f960be065de7 namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: 1fddafca-1acb-484a-bcdf-227f552e064b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-27640 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: f9d35da1-2e5e-457c-bf3e-312931906ade kind: Event lastTimestamp: "2026-08-05T17:37:26Z" message: InferenceGraph [sequence-graph-27640] is Ready metadata: creationTimestamp: "2026-08-05T17:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:26Z" name: sequence-graph-27640.18c8f960c3f0caed namespace: kserve-ci-e2e-test resourceVersion: "21617" uid: e1d9ff36-ea3f-4ead-b61e-a377c71bfbb7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:28:11.249310Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d9755-5544975675-kch4r namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: a265111b-fd39-47a5-bde4-7dec632e7dfc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-d9755-5544975675-kch4r to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:28:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:28:11Z" name: sequence-graph-d9755-5544975675-kch4r.18c8f8df8b352e65 namespace: kserve-ci-e2e-test resourceVersion: "17146" uid: 37b1e93a-a60a-4903-9a99-095b1856a9e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:28:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d9755-5544975675-kch4r namespace: kserve-ci-e2e-test resourceVersion: "17149" uid: a265111b-fd39-47a5-bde4-7dec632e7dfc kind: Event lastTimestamp: "2026-08-05T17:28:11Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:28:11Z" name: sequence-graph-d9755-5544975675-kch4r.18c8f8dfa460243c namespace: kserve-ci-e2e-test resourceVersion: "17162" uid: 6408b8c4-68b1-48ba-b081-41fefa1995f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:28:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d9755} kind: Pod name: sequence-graph-d9755-5544975675-kch4r namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: a265111b-fd39-47a5-bde4-7dec632e7dfc kind: Event lastTimestamp: "2026-08-05T17:28:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:28:11Z" name: sequence-graph-d9755-5544975675-kch4r.18c8f8dfa59481de namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: aacc8f6b-9081-4058-b873-bf1c70027f07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:28:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d9755} kind: Pod name: sequence-graph-d9755-5544975675-kch4r namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: a265111b-fd39-47a5-bde4-7dec632e7dfc kind: Event lastTimestamp: "2026-08-05T17:28:11Z" message: 'Created container: sequence-graph-d9755' metadata: creationTimestamp: "2026-08-05T17:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:28:11Z" name: sequence-graph-d9755-5544975675-kch4r.18c8f8dfa869ef09 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 605dc431-91aa-4b99-8ef1-62741f82aec0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:28:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d9755} kind: Pod name: sequence-graph-d9755-5544975675-kch4r namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: a265111b-fd39-47a5-bde4-7dec632e7dfc kind: Event lastTimestamp: "2026-08-05T17:28:11Z" message: Started container sequence-graph-d9755 metadata: creationTimestamp: "2026-08-05T17:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:28:11Z" name: sequence-graph-d9755-5544975675-kch4r.18c8f8dfa8d7393b namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: 2f5b8cc7-1a25-47ba-a85d-bf77c9dda0fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d9755} kind: Pod name: sequence-graph-d9755-5544975675-kch4r namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: a265111b-fd39-47a5-bde4-7dec632e7dfc kind: Event lastTimestamp: "2026-08-05T17:36:25Z" message: Stopping container sequence-graph-d9755 metadata: creationTimestamp: "2026-08-05T17:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:25Z" name: sequence-graph-d9755-5544975675-kch4r.18c8f952b8c6290a namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: 8e9742c9-dccb-4dd0-8d2f-e9c8191fb198 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d9755} kind: Pod name: sequence-graph-d9755-5544975675-kch4r namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: a265111b-fd39-47a5-bde4-7dec632e7dfc kind: Event lastTimestamp: "2026-08-05T17:36:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:52Z" name: sequence-graph-d9755-5544975675-kch4r.18c8f953339a26e0 namespace: kserve-ci-e2e-test resourceVersion: "21303" uid: 1cbf4274-5f25-427f-b442-ddab77927a4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:28:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-d9755-5544975675 namespace: kserve-ci-e2e-test resourceVersion: "17135" uid: 193abbce-5318-4016-a261-120814228364 kind: Event lastTimestamp: "2026-08-05T17:28:11Z" message: 'Created pod: sequence-graph-d9755-5544975675-kch4r' metadata: creationTimestamp: "2026-08-05T17:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:28:11Z" name: sequence-graph-d9755-5544975675.18c8f8df8abac1c1 namespace: kserve-ci-e2e-test resourceVersion: "17142" uid: 913a0701-a31a-424b-ba8a-ba07a81489f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:28:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-d9755 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: eb62381a-94b2-46e5-83e6-dd98fa68d853 kind: Event lastTimestamp: "2026-08-05T17:28:11Z" message: Scaled up replica set sequence-graph-d9755-5544975675 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:28:11Z" name: sequence-graph-d9755.18c8f8df89c461fc namespace: kserve-ci-e2e-test resourceVersion: "17137" uid: ac149e17-6b63-4c71-99e2-3b2df05fbdb0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:28:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d9755 namespace: kserve-ci-e2e-test resourceVersion: "17153" uid: 11f92d0b-0a98-418f-9771-d7350c2c639a kind: Event lastTimestamp: "2026-08-05T17:28:11Z" message: InferenceGraph [sequence-graph-d9755] is Ready metadata: creationTimestamp: "2026-08-05T17:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:28:11Z" name: sequence-graph-d9755.18c8f8df8f83d906 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 54baa26e-045e-4ae3-92d3-6f6abe5f60db reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d9755 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: d72daefd-5173-4486-801a-189ee0ce10ea kind: Event lastTimestamp: "2026-08-05T17:28:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:28:41Z" name: sequence-graph-d9755.18c8f8e3086a7eda namespace: kserve-ci-e2e-test resourceVersion: "17381" uid: d1cfbbf2-c885-4274-bef6-7670e5cf2bf4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d9755 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: d72daefd-5173-4486-801a-189ee0ce10ea kind: Event lastTimestamp: "2026-08-05T17:28:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:28:41Z" name: sequence-graph-d9755.18c8f8e308d56767 namespace: kserve-ci-e2e-test resourceVersion: "17382" uid: 5bf5c994-a4c7-4a93-84be-f07cbec27c2b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:38:36.555723Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e149b-85d74f945f-mjk8l namespace: kserve-ci-e2e-test resourceVersion: "22396" uid: 8bf18f3a-6dc8-41ce-b6ed-eed260aca688 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e149b-85d74f945f-mjk8l to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:38:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:38:36Z" name: sequence-graph-e149b-85d74f945f-mjk8l.18c8f971225fe044 namespace: kserve-ci-e2e-test resourceVersion: "22401" uid: 108f9931-1330-4ad2-bf46-3d0c4bb828eb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e149b-85d74f945f-mjk8l namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 8bf18f3a-6dc8-41ce-b6ed-eed260aca688 kind: Event lastTimestamp: "2026-08-05T17:38:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-e149b-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:38:36Z" name: sequence-graph-e149b-85d74f945f-mjk8l.18c8f9712b109761 namespace: kserve-ci-e2e-test resourceVersion: "22412" uid: b476579b-94e6-4e69-82ce-cb5e9f2a2d1f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e149b-85d74f945f-mjk8l namespace: kserve-ci-e2e-test resourceVersion: "22405" uid: 8bf18f3a-6dc8-41ce-b6ed-eed260aca688 kind: Event lastTimestamp: "2026-08-05T17:38:37Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:38:37Z" name: sequence-graph-e149b-85d74f945f-mjk8l.18c8f9715ff7518d namespace: kserve-ci-e2e-test resourceVersion: "22422" uid: b457659e-50b6-473a-be3e-0792686c18b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e149b} kind: Pod name: sequence-graph-e149b-85d74f945f-mjk8l namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 8bf18f3a-6dc8-41ce-b6ed-eed260aca688 kind: Event lastTimestamp: "2026-08-05T17:38:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:38:37Z" name: sequence-graph-e149b-85d74f945f-mjk8l.18c8f97161c07ada namespace: kserve-ci-e2e-test resourceVersion: "22424" uid: 90e1b8a1-ded5-4135-af16-a5269a9e68cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e149b} kind: Pod name: sequence-graph-e149b-85d74f945f-mjk8l namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 8bf18f3a-6dc8-41ce-b6ed-eed260aca688 kind: Event lastTimestamp: "2026-08-05T17:38:37Z" message: 'Created container: sequence-graph-e149b' metadata: creationTimestamp: "2026-08-05T17:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:38:37Z" name: sequence-graph-e149b-85d74f945f-mjk8l.18c8f9716561cb4b namespace: kserve-ci-e2e-test resourceVersion: "22425" uid: c160da7d-fc2b-4f2f-a2ca-6d2d054f8d35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e149b} kind: Pod name: sequence-graph-e149b-85d74f945f-mjk8l namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 8bf18f3a-6dc8-41ce-b6ed-eed260aca688 kind: Event lastTimestamp: "2026-08-05T17:38:37Z" message: Started container sequence-graph-e149b metadata: creationTimestamp: "2026-08-05T17:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:38:37Z" name: sequence-graph-e149b-85d74f945f-mjk8l.18c8f97165d34ba3 namespace: kserve-ci-e2e-test resourceVersion: "22426" uid: 41f26210-d39b-43bf-8c07-c0d3d01d8414 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e149b} kind: Pod name: sequence-graph-e149b-85d74f945f-mjk8l namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 8bf18f3a-6dc8-41ce-b6ed-eed260aca688 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Stopping container sequence-graph-e149b metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: sequence-graph-e149b-85d74f945f-mjk8l.18c8f9e44863dfc8 namespace: kserve-ci-e2e-test resourceVersion: "26332" uid: 5445931a-79f1-4c59-a995-dd1cacd691af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e149b} kind: Pod name: sequence-graph-e149b-85d74f945f-mjk8l namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 8bf18f3a-6dc8-41ce-b6ed-eed260aca688 kind: Event lastTimestamp: "2026-08-05T17:47:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:19Z" name: sequence-graph-e149b-85d74f945f-mjk8l.18c8f9e5146f10e1 namespace: kserve-ci-e2e-test resourceVersion: "26881" uid: 7c1ef657-d3b9-4c10-a8be-e9916aebd8cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e149b-85d74f945f namespace: kserve-ci-e2e-test resourceVersion: "22392" uid: 03edfada-2f4c-4553-92d2-aed35570eb8c kind: Event lastTimestamp: "2026-08-05T17:38:36Z" message: 'Created pod: sequence-graph-e149b-85d74f945f-mjk8l' metadata: creationTimestamp: "2026-08-05T17:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:36Z" name: sequence-graph-e149b-85d74f945f.18c8f97122019ffc namespace: kserve-ci-e2e-test resourceVersion: "22398" uid: fac02eda-d5a4-4614-b2c2-329ed396fea5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e149b namespace: kserve-ci-e2e-test resourceVersion: "22388" uid: 9785f1dc-d061-41cc-944e-9f3a38d206fd kind: Event lastTimestamp: "2026-08-05T17:38:36Z" message: Scaled up replica set sequence-graph-e149b-85d74f945f from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:36Z" name: sequence-graph-e149b.18c8f971212b5182 namespace: kserve-ci-e2e-test resourceVersion: "22394" uid: b6c9c715-d757-4a75-8f24-73dd1d86d460 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e149b namespace: kserve-ci-e2e-test resourceVersion: "22409" uid: 191492dc-c9b5-438b-82e6-86bd1467905e kind: Event lastTimestamp: "2026-08-05T17:38:36Z" message: InferenceGraph [sequence-graph-e149b] is Ready metadata: creationTimestamp: "2026-08-05T17:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:38:36Z" name: sequence-graph-e149b.18c8f9712a0cc406 namespace: kserve-ci-e2e-test resourceVersion: "22411" uid: 0e212fcf-3741-44c7-a6b5-11ef3667924a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:38:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e149b namespace: kserve-ci-e2e-test resourceVersion: "22391" uid: 822d84ba-b1fd-47c4-bdc9-1b551f0fa583 kind: Event lastTimestamp: "2026-08-05T17:39:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:39:06Z" name: sequence-graph-e149b.18c8f9749f8a821b namespace: kserve-ci-e2e-test resourceVersion: "22638" uid: bd4bc1a0-4532-41f0-aaab-facd1d3d8fc5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:38:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e149b namespace: kserve-ci-e2e-test resourceVersion: "22391" uid: 822d84ba-b1fd-47c4-bdc9-1b551f0fa583 kind: Event lastTimestamp: "2026-08-05T17:39:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:39:06Z" name: sequence-graph-e149b.18c8f9749fd07fc8 namespace: kserve-ci-e2e-test resourceVersion: "22639" uid: bb9af4be-0cf3-430a-b0e7-cceddfefc6fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:46:51.374071Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-92074-768886f666-wd429 namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: cc80bf0e-91ce-4e32-91f3-1fdb92e74706 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-92074-768886f666-wd429 to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:46:51Z" name: splitter-graph-92074-768886f666-wd429.18c8f9e457d8ae16 namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: bf135901-af2c-40e8-a28a-3bf6c1235a9e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-92074-768886f666-wd429 namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: cc80bf0e-91ce-4e32-91f3-1fdb92e74706 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:46:51Z" name: splitter-graph-92074-768886f666-wd429.18c8f9e471bf56e8 namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: fbe86304-34c9-404f-aaca-15845e485626 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-92074} kind: Pod name: splitter-graph-92074-768886f666-wd429 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: cc80bf0e-91ce-4e32-91f3-1fdb92e74706 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: splitter-graph-92074-768886f666-wd429.18c8f9e4730a0117 namespace: kserve-ci-e2e-test resourceVersion: "26470" uid: 5f51f0a9-068f-4283-86e5-ec6d410fc270 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-92074} kind: Pod name: splitter-graph-92074-768886f666-wd429 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: cc80bf0e-91ce-4e32-91f3-1fdb92e74706 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'Created container: splitter-graph-92074' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: splitter-graph-92074-768886f666-wd429.18c8f9e47639517a namespace: kserve-ci-e2e-test resourceVersion: "26476" uid: cfba5629-6b95-4cee-b7f3-d47d21b40639 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-92074} kind: Pod name: splitter-graph-92074-768886f666-wd429 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: cc80bf0e-91ce-4e32-91f3-1fdb92e74706 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Started container splitter-graph-92074 metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: splitter-graph-92074-768886f666-wd429.18c8f9e476a7ce4d namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: c856e240-642f-4bba-8b9f-a0466bbe678f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-92074} kind: Pod name: splitter-graph-92074-768886f666-wd429 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: cc80bf0e-91ce-4e32-91f3-1fdb92e74706 kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: Stopping container splitter-graph-92074 metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:01Z" name: splitter-graph-92074-768886f666-wd429.18c8f9e6af1dd087 namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: 3f58734a-de52-423a-8ef3-cdce314e0d85 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:47:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-92074} kind: Pod name: splitter-graph-92074-768886f666-wd429 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: cc80bf0e-91ce-4e32-91f3-1fdb92e74706 kind: Event lastTimestamp: "2026-08-05T17:47:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:28Z" name: splitter-graph-92074-768886f666-wd429.18c8f9e72e5107b2 namespace: kserve-ci-e2e-test resourceVersion: "26951" uid: 86112605-8667-4378-9350-3d2d21982611 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-92074-768886f666 namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: 1e621ca3-e370-4e43-ac5a-053cef7c0d09 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'Created pod: splitter-graph-92074-768886f666-wd429' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: splitter-graph-92074-768886f666.18c8f9e457349d4b namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: d41bd617-dc62-4469-b4cc-20ca9f766234 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-92074 namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 9ed0903f-3929-4553-99c7-b4983b2eedbf kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Scaled up replica set splitter-graph-92074-768886f666 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: splitter-graph-92074.18c8f9e4565e44f8 namespace: kserve-ci-e2e-test resourceVersion: "26405" uid: d093a071-13c0-4c70-a28c-34299a85c143 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-92074 namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: 865f70e9-c954-4eb1-a199-345e03efcd21 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: InferenceGraph [splitter-graph-92074] is Ready metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:46:51Z" name: splitter-graph-92074.18c8f9e45d9cc4a4 namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: c7bd7755-e1da-49f6-aac1-ede72097a61b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:48:01.694156Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-db0ba-5f955c994d-pjzfw namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: 9871ba28-54a9-4c9f-8ec8-8fd4d79c3b9b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-db0ba-5f955c994d-pjzfw to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:48:01Z" name: splitter-graph-db0ba-5f955c994d-pjzfw.18c8f9f4b73ffaa8 namespace: kserve-ci-e2e-test resourceVersion: "27290" uid: 583d7b65-4642-46fa-ae08-d35a77145ba7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-db0ba-5f955c994d-pjzfw namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: 9871ba28-54a9-4c9f-8ec8-8fd4d79c3b9b kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-db0ba-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:48:01Z" name: splitter-graph-db0ba-5f955c994d-pjzfw.18c8f9f4c14d60f2 namespace: kserve-ci-e2e-test resourceVersion: "27308" uid: 8b408a0d-9751-48ca-9665-eaa5e21e61e8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:02Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-db0ba-5f955c994d-pjzfw namespace: kserve-ci-e2e-test resourceVersion: "27295" uid: 9871ba28-54a9-4c9f-8ec8-8fd4d79c3b9b kind: Event lastTimestamp: "2026-08-05T17:48:02Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:48:02Z" name: splitter-graph-db0ba-5f955c994d-pjzfw.18c8f9f4f475fb34 namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: 11117e2f-6af2-4f19-b449-ba02f3cdbf8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-db0ba} kind: Pod name: splitter-graph-db0ba-5f955c994d-pjzfw namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: 9871ba28-54a9-4c9f-8ec8-8fd4d79c3b9b kind: Event lastTimestamp: "2026-08-05T17:48:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:48:02Z" name: splitter-graph-db0ba-5f955c994d-pjzfw.18c8f9f4f5db9099 namespace: kserve-ci-e2e-test resourceVersion: "27322" uid: 9da76d79-ba12-415d-be03-47a4789aeb4f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-db0ba} kind: Pod name: splitter-graph-db0ba-5f955c994d-pjzfw namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: 9871ba28-54a9-4c9f-8ec8-8fd4d79c3b9b kind: Event lastTimestamp: "2026-08-05T17:48:02Z" message: 'Created container: splitter-graph-db0ba' metadata: creationTimestamp: "2026-08-05T17:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:48:02Z" name: splitter-graph-db0ba-5f955c994d-pjzfw.18c8f9f4f8cd4de3 namespace: kserve-ci-e2e-test resourceVersion: "27323" uid: 6af63c6b-6cf9-45a1-b284-ab25bef05767 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-db0ba} kind: Pod name: splitter-graph-db0ba-5f955c994d-pjzfw namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: 9871ba28-54a9-4c9f-8ec8-8fd4d79c3b9b kind: Event lastTimestamp: "2026-08-05T17:48:02Z" message: Started container splitter-graph-db0ba metadata: creationTimestamp: "2026-08-05T17:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:48:02Z" name: splitter-graph-db0ba-5f955c994d-pjzfw.18c8f9f4f941eaf2 namespace: kserve-ci-e2e-test resourceVersion: "27324" uid: b58588cd-7b35-427c-ba2e-c08b36fd9844 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-db0ba} kind: Pod name: splitter-graph-db0ba-5f955c994d-pjzfw namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: 9871ba28-54a9-4c9f-8ec8-8fd4d79c3b9b kind: Event lastTimestamp: "2026-08-05T17:56:16Z" message: Stopping container splitter-graph-db0ba metadata: creationTimestamp: "2026-08-05T17:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:56:16Z" name: splitter-graph-db0ba-5f955c994d-pjzfw.18c8fa67e1f8a623 namespace: kserve-ci-e2e-test resourceVersion: "30776" uid: 82157fe9-7090-4215-b4fb-3819bea11059 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-db0ba} kind: Pod name: splitter-graph-db0ba-5f955c994d-pjzfw namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: 9871ba28-54a9-4c9f-8ec8-8fd4d79c3b9b kind: Event lastTimestamp: "2026-08-05T17:56:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:56:43Z" name: splitter-graph-db0ba-5f955c994d-pjzfw.18c8fa68788984c8 namespace: kserve-ci-e2e-test resourceVersion: "31007" uid: cb005233-fbea-41e0-97ad-caf70d006ab2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-db0ba-5f955c994d namespace: kserve-ci-e2e-test resourceVersion: "27279" uid: c3206308-f793-44e2-8983-92099be15436 kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: 'Created pod: splitter-graph-db0ba-5f955c994d-pjzfw' metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:01Z" name: splitter-graph-db0ba-5f955c994d.18c8f9f4b6a8075f namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: fd710f2e-85ef-4adc-b351-491d27e0adfb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-db0ba namespace: kserve-ci-e2e-test resourceVersion: "27278" uid: 80ea3eff-e67f-453c-85f4-8c1ad26f261a kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: Scaled up replica set splitter-graph-db0ba-5f955c994d from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:01Z" name: splitter-graph-db0ba.18c8f9f4b5e50af2 namespace: kserve-ci-e2e-test resourceVersion: "27281" uid: bb7aef3c-11b2-4a7d-82f3-7bb42a0d0e1d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-db0ba namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: 20e80df2-248b-48f1-a985-37e159f5b0e5 kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: InferenceGraph [splitter-graph-db0ba] is Ready metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:48:01Z" name: splitter-graph-db0ba.18c8f9f4b9cf5135 namespace: kserve-ci-e2e-test resourceVersion: "27300" uid: 0ffbe9cd-da48-4c04-b709-99a375764fac reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:48:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-db0ba namespace: kserve-ci-e2e-test resourceVersion: "27280" uid: 8dffd78e-0730-45cd-8398-7d12697cac0a kind: Event lastTimestamp: "2026-08-05T17:48:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:46Z" name: splitter-graph-db0ba.18c8f9f834b47c62 namespace: kserve-ci-e2e-test resourceVersion: "27663" uid: 513e6aee-7118-45e4-aced-a53de1857844 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:48:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-db0ba namespace: kserve-ci-e2e-test resourceVersion: "27280" uid: 8dffd78e-0730-45cd-8398-7d12697cac0a kind: Event lastTimestamp: "2026-08-05T17:48:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:46Z" name: splitter-graph-db0ba.18c8f9f835283f85 namespace: kserve-ci-e2e-test resourceVersion: "27664" uid: 3d044aa8-3f1e-4612-8712-d9d9e25d24e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:36:26.152078Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20818" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-27640-predictor-55c9498c7d-ll287 to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f952c5b62512 namespace: kserve-ci-e2e-test resourceVersion: "20823" uid: 5fa1d12e-dfea-4335-86a3-d8cfd99c222e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-27640-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f952d42d82f3 namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: bc4dee4e-8bf7-438b-a4d5-8f898db32b7c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:36:27Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:36:27Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f95302f4ab61 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: c079b010-de61-478a-b8bf-ce79bc011deb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:36:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:27Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f95304787cdd namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: cd3cb7dd-1a5d-4181-b835-1eae500aea0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:36:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:27Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f9530822fdd2 namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 3206951e-32fa-42b2-aefc-33a336bf51be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:36:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:27Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f95308906ee7 namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: d943ae07-937b-434f-a1ad-ab0e1002b9cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:36:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:27Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f953089a8b59 namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: f8de6f72-554f-4609-bdbd-9ea2b4b84738 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:36:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:27Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f9530b5d7dc0 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 78d1d9f4-1cf2-472e-8be8-af91af89ee7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:36:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:27Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f9530bcbc761 namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: c2792e83-50aa-4163-b6c7-fd246df351f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:37:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:03Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f9532f5c4df0 namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: 0aff5ca0-76be-4ab4-b619-711ade3b65fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f96324c847ad namespace: kserve-ci-e2e-test resourceVersion: "21769" uid: 9336ab71-fd3b-4d8e-acd0-bdea31390f42 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f96324c8d260 namespace: kserve-ci-e2e-test resourceVersion: "21771" uid: fc29bd5f-3113-41d9-9cf2-8f9cebd38509 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27640-predictor-55c9498c7d-ll287 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: aee60f09-ffd8-4733-b965-a572468b235c kind: Event lastTimestamp: "2026-08-05T17:37:38Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:38Z" name: success-200-isvc-27640-predictor-55c9498c7d-ll287.18c8f963b781d0f3 namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: 8bfc665e-780e-461e-a044-947388067363 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-27640-predictor-55c9498c7d namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: d1803c26-86e5-424e-959a-caa09852f4df kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'Created pod: success-200-isvc-27640-predictor-55c9498c7d-ll287' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-27640-predictor-55c9498c7d.18c8f952c52b178a namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: f7a26e31-adc5-4b9a-bac7-16c198bd6eb8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: f55365d0-7fc3-4bb9-b652-19fb16790ddb kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Scaled up replica set success-200-isvc-27640-predictor-55c9498c7d from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-27640-predictor.18c8f952c2d484cd namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 79656f89-f35f-4f68-b09f-6c36f4fdb9c7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: 7e074efe-f516-4899-a2f4-adc5a2402d49 kind: Event lastTimestamp: "2026-08-05T17:36:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:41Z" name: success-200-isvc-27640-predictor.18c8f95641fe3f54 namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: d10dae79-f764-4368-8bab-9eb655e05135 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: 7e074efe-f516-4899-a2f4-adc5a2402d49 kind: Event lastTimestamp: "2026-08-05T17:36:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:41Z" name: success-200-isvc-27640-predictor.18c8f956426d9870 namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: ca191b31-9440-4f25-ba60-4513dd216701 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:36:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: 7e074efe-f516-4899-a2f4-adc5a2402d49 kind: Event lastTimestamp: "2026-08-05T17:37:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:26Z" name: success-200-isvc-27640-predictor.18c8f959c131cd39 namespace: kserve-ci-e2e-test resourceVersion: "21610" uid: eb143a89-439e-4c33-85c8-10426a48ce3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:36:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-27640-predictor namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: 7e074efe-f516-4899-a2f4-adc5a2402d49 kind: Event lastTimestamp: "2026-08-05T17:37:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:26Z" name: success-200-isvc-27640-predictor.18c8f959c1ab5d5a namespace: kserve-ci-e2e-test resourceVersion: "21612" uid: df65277b-2af9-4fbc-8538-be60d31900ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-27640 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 52e228e5-b53f-4349-81a0-3960e432d151 kind: Event lastTimestamp: "2026-08-05T17:37:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:14Z" name: success-200-isvc-27640.18c8f952c2dbc750 namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: bcfa2788-9f48-45ac-bf12-1226a625ebdb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-27640 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 52e228e5-b53f-4349-81a0-3960e432d151 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-27640": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-27640": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-27640.18c8f952c3fd0cdc namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 983c1e09-f1b4-4eb8-ab1a-2997d35e296e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-27640 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 52e228e5-b53f-4349-81a0-3960e432d151 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-27640": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-27640.18c8f952c3fd4f72 namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: db8acf47-3385-41c6-8c37-b7f101fc2db3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-27640 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 52e228e5-b53f-4349-81a0-3960e432d151 kind: Event lastTimestamp: "2026-08-05T17:37:14Z" message: InferenceService [success-200-isvc-27640] is Ready metadata: creationTimestamp: "2026-08-05T17:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:14Z" name: success-200-isvc-27640.18c8f95deb2aeae6 namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 4f35028d-83b8-4734-befe-b14081ef4aed reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:46:51.332733Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-741a0-predictor-77484d674-xt592 to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8f9e45561e527 namespace: kserve-ci-e2e-test resourceVersion: "26397" uid: d9ec996c-fc83-48fb-98bc-677f2d0b9cfa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8f9e46f2293bf namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: 0cc79e28-c073-44f6-8a15-bbfb10202625 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8f9e47084c643 namespace: kserve-ci-e2e-test resourceVersion: "26466" uid: 9914e23b-94b8-4fb2-a50e-bfa2d5173a6d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8f9e473be93f9 namespace: kserve-ci-e2e-test resourceVersion: "26471" uid: 7a700e01-fd50-4a69-ad92-d433529668c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8f9e47432ab3e namespace: kserve-ci-e2e-test resourceVersion: "26472" uid: 929525d8-1427-46a9-a0e1-a44dc5cbdbd7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8f9e4743ea43d namespace: kserve-ci-e2e-test resourceVersion: "26473" uid: 596f7dab-575b-46cb-99fb-dfb783aead50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8f9e477a2b31b namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: 9533348a-7a1d-44d2-8d74-33f6db74e51a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8f9e478101301 namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: 1bf24d07-8a35-40a4-bbe7-37248c4b69fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-05T17:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T17:47:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:38Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8f9e4a1b5d771 namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: dede281c-654f-4234-b98b-f8a1c2b96b0b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T18:04:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T18:04:20Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8fad8a726424d namespace: kserve-ci-e2e-test resourceVersion: "34147" uid: f4f1fc72-2fd0-487a-828e-4c8be4806214 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T18:04:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T18:04:20Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8fad8a726aba0 namespace: kserve-ci-e2e-test resourceVersion: "34148" uid: 82805971-c3cc-4ce5-9536-0ed59925f3c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T18:04:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-741a0-predictor-77484d674-xt592 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 30ca866f-36b3-4229-9126-d138dd30da9f kind: Event lastTimestamp: "2026-08-05T18:04:23Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T18:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T18:04:23Z" name: success-200-isvc-741a0-predictor-77484d674-xt592.18c8fad9568c7acd namespace: kserve-ci-e2e-test resourceVersion: "34217" uid: 9d069a76-16db-4047-9664-e5217e6455c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-741a0-predictor-77484d674 namespace: kserve-ci-e2e-test resourceVersion: "26369" uid: 55ed3366-2446-448d-bc72-037c4cf4cd83 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'Created pod: success-200-isvc-741a0-predictor-77484d674-xt592' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor-77484d674.18c8f9e454c336b1 namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: 89834bcd-53a9-4a9d-ba4d-289cf20e8cad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26363" uid: 01f2c5e5-843f-4dda-b4dc-2ac192af7e2d kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Scaled up replica set success-200-isvc-741a0-predictor-77484d674 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0-predictor.18c8f9e4522fbce1 namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: 99d1b442-f9b0-4bfb-9e19-503125a08061 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:47:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: 5230c342-574a-42b4-9f53-b40f6dfe88f7 kind: Event lastTimestamp: "2026-08-05T17:47:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:21Z" name: success-200-isvc-741a0-predictor.18c8f9e7d163fa3d namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: dcc2dabc-947d-41f3-88e9-ebadaca80f5c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:47:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: 5230c342-574a-42b4-9f53-b40f6dfe88f7 kind: Event lastTimestamp: "2026-08-05T17:47:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:21Z" name: success-200-isvc-741a0-predictor.18c8f9e7d1de155b namespace: kserve-ci-e2e-test resourceVersion: "26899" uid: 97ae2bd8-1841-4a9d-bb76-f654be1c604a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:47:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 5230c342-574a-42b4-9f53-b40f6dfe88f7 kind: Event lastTimestamp: "2026-08-05T17:48:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:06Z" name: success-200-isvc-741a0-predictor.18c8f9eecf7f4a8d namespace: kserve-ci-e2e-test resourceVersion: "27356" uid: 20cd7a34-d69c-4ae3-a211-320264a124f8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:47:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-741a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 5230c342-574a-42b4-9f53-b40f6dfe88f7 kind: Event lastTimestamp: "2026-08-05T17:48:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:06Z" name: success-200-isvc-741a0-predictor.18c8f9eecfed0119 namespace: kserve-ci-e2e-test resourceVersion: "27358" uid: 755e4eab-0b6b-4a73-81eb-05b6a41abfd4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-741a0 namespace: kserve-ci-e2e-test resourceVersion: "26338" uid: 12bc90aa-5464-4c1b-a102-24300f99a894 kind: Event lastTimestamp: "2026-08-05T17:47:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:48Z" name: success-200-isvc-741a0.18c8f9e452831088 namespace: kserve-ci-e2e-test resourceVersion: "27135" uid: 860814b9-c935-47ef-b7b6-2981dbea9b4c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-741a0 namespace: kserve-ci-e2e-test resourceVersion: "26338" uid: 12bc90aa-5464-4c1b-a102-24300f99a894 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-741a0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-741a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0.18c8f9e45324ad67 namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 9a2a84c4-ab78-411f-828b-a0c82af81bdc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-741a0 namespace: kserve-ci-e2e-test resourceVersion: "26338" uid: 12bc90aa-5464-4c1b-a102-24300f99a894 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-741a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-741a0.18c8f9e45325022f namespace: kserve-ci-e2e-test resourceVersion: "26384" uid: 50946c0b-ff25-4eba-a4af-6a0e7cdfba79 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-741a0 namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 12bc90aa-5464-4c1b-a102-24300f99a894 kind: Event lastTimestamp: "2026-08-05T17:47:48Z" message: InferenceService [success-200-isvc-741a0] is Ready metadata: creationTimestamp: "2026-08-05T17:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:48Z" name: success-200-isvc-741a0.18c8f9f1b18cd7d6 namespace: kserve-ci-e2e-test resourceVersion: "27134" uid: 9250b03e-d8d7-4db9-85a0-f37f87bf7def reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:24:41.094551Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-829a9-predictor-54f97d665d-jlvfn to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:24:41Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8ae9d020922 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: ce4ec5f6-147b-46d8-98db-596e61fcdf1e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:24:41Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8aeb6879b63 namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: 97b3bc41-f1cf-43e7-bf5e-a145b5184ddf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:41Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8aeb80f7860 namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 70c3bc0f-f63a-4b6b-a994-fa82c04e3f1a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:24:57Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" in 16.102s (16.103s including waiting). Image size: 1334934628 bytes.' metadata: creationTimestamp: "2026-08-05T17:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:57Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8b277e040c8 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 3114e16b-be37-48fc-b6ac-a815a23eb6a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:24:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:57Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8b27be27388 namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: 546e17ff-38ce-4d68-8b2c-36b99db0f81a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:24:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:57Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8b27c5df0f1 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 5e463037-229e-4701-9dc8-5cb02825557d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:24:57Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-08-05T17:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:24:57Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8b27c6b4a0f namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: 4a6ec563-18b9-401d-bff6-e8f56afeabb6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:25:00Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.555s (2.555s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-08-05T17:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:00Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8b314bded29 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 6119ec96-4010-4fad-baab-8f73292aaa93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:25:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:00Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8b317d01413 namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 494b95fe-98ca-4b20-8fa5-424904162369 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:25:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:00Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8b31851bc85 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 7ebb4ded-c798-487b-b130-987c613de0d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:25:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:25:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:25:37Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8b33fefb857 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 7c277f12-8ed7-4e4c-b57b-b947311ea23b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8c39a19715a namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 11afc657-1d71-4993-a8a1-5b3212f04bba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8c39a1a79db namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: 9ada51c6-ea98-478b-a323-867788eb33e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 54086e09-ff4e-4168-94f9-58df784d7811 kind: Event lastTimestamp: "2026-08-05T17:26:12Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:12Z" name: success-200-isvc-829a9-predictor-54f97d665d-jlvfn.18c8f8c3c83769a8 namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: fcd9ed46-d5c4-4c6d-9e91-72a78e8d7b8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-829a9-predictor-54f97d665d namespace: kserve-ci-e2e-test resourceVersion: "14727" uid: e1213848-3ffc-4a8d-b7f9-434d80b75e63 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: 'Created pod: success-200-isvc-829a9-predictor-54f97d665d-jlvfn' metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:41Z" name: success-200-isvc-829a9-predictor-54f97d665d.18c8f8ae9c75aa68 namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 728a4477-fa68-4a51-ae54-bb2caaa67419 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: e0eda4b1-f172-4f31-8cd3-c382aaf56ac7 kind: Event lastTimestamp: "2026-08-05T17:24:41Z" message: Scaled up replica set success-200-isvc-829a9-predictor-54f97d665d from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:24:41Z" name: success-200-isvc-829a9-predictor.18c8f8ae9aa05611 namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: aae816bc-9973-4905-bcf3-4e4c915595ee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: 8c48f8c3-699e-4537-8fbe-3d6e7313b9f2 kind: Event lastTimestamp: "2026-08-05T17:25:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:41Z" name: success-200-isvc-829a9-predictor.18c8f8b21925fd09 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 4f0b4f7d-e004-4486-9bde-f47fd2a5c84a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: 8c48f8c3-699e-4537-8fbe-3d6e7313b9f2 kind: Event lastTimestamp: "2026-08-05T17:25:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:25:41Z" name: success-200-isvc-829a9-predictor.18c8f8b219974a31 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: ee3dccfd-ae00-40bc-a374-40edfa0eaea9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:25:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 8c48f8c3-699e-4537-8fbe-3d6e7313b9f2 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:11Z" name: success-200-isvc-829a9-predictor.18c8f8c014ebe1e0 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 5ecac9dd-75b2-4658-b72b-98f507bbabd7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:25:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-829a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 8c48f8c3-699e-4537-8fbe-3d6e7313b9f2 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:11Z" name: success-200-isvc-829a9-predictor.18c8f8c015432e98 namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: a9cc3035-ee4d-4948-b7d3-1cb855adc63d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:24:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-829a9 namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 23b1006d-101c-4289-8e56-5120479703df kind: Event lastTimestamp: "2026-08-05T17:25:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:25:47Z" name: success-200-isvc-829a9.18c8f8ae9aa5aa11 namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 485f1248-6982-473d-893a-b16799395d1f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-829a9 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 23b1006d-101c-4289-8e56-5120479703df kind: Event lastTimestamp: "2026-08-05T17:25:47Z" message: InferenceService [success-200-isvc-829a9] is Ready metadata: creationTimestamp: "2026-08-05T17:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:25:47Z" name: success-200-isvc-829a9.18c8f8bdfe4c104f namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 83521708-710c-42b6-8953-5f30d9b332d4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:36:36.488309Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f9552dcc928b namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: 92c03c3b-7293-42e8-8a3e-494de5520e9a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f9554759c941 namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 2c93c6f3-18df-47eb-a3e9-2ceae80709d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f95548b4ab8e namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: 1f9cfccc-03f5-41b7-a6e0-84cd27cc31d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f9554c16b2ba namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 005c0725-b723-48b4-aca7-c32dfa57ddb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f9554c9f0754 namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: d69cd9df-9123-4f79-a566-e87d4fa0a191 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f9554ca8daaa namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 772a3c45-66f4-4912-9c98-16051655e8b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f9554fb467f7 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: d27cf862-1586-4905-abad-d854747ed816 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:36:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:37Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f955502704c9 namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 12f4b196-6ebf-4d4c-940e-bab8190de210 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-05T17:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:45:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:54Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f95586cd4639 namespace: kserve-ci-e2e-test resourceVersion: "25855" uid: 5f43c0b2-b031-41ba-a1a5-8635e9d6ed04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f9d65bfc7380 namespace: kserve-ci-e2e-test resourceVersion: "25736" uid: 83ddd8b9-8a4d-4048-b550-cfc82ce656ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f9d65bfcf14b namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: 83adbed7-9c59-4bc0-8fff-41a4ee62f7d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fbe776c3-891d-4fbf-acf1-6f4b72abda6c kind: Event lastTimestamp: "2026-08-05T17:45:53Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:53Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz.18c8f9d6fb4fda19 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: f0b0110a-d7ca-4db0-85bf-44c70fd806c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8dddc-predictor-66fd6cdf4 namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: b98f55d6-adec-4fe4-80a9-d43e6793d5f4 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: 'Created pod: success-200-isvc-8dddc-predictor-66fd6cdf4-znnzz' metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-8dddc-predictor-66fd6cdf4.18c8f9552d530a75 namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: af9d203b-6bb8-4e29-9cd3-d053d1ccb561 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: 2e075321-19b8-450c-914d-1a09cbbf8a47 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: Scaled up replica set success-200-isvc-8dddc-predictor-66fd6cdf4 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-8dddc-predictor.18c8f9552a860c2c namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 55272b16-46a7-4ba1-a2bf-6cf4133cc3c8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:36:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: faefcd91-0895-4351-8f91-10804db684a2 kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-8dddc-predictor.18c8f958a93e0108 namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: 443ad829-b446-49eb-9fa9-c09a7d6c6bdf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:36:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: faefcd91-0895-4351-8f91-10804db684a2 kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-8dddc-predictor.18c8f958a98428e5 namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 4a8437e1-15f8-456d-816b-d2e98a453502 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: faefcd91-0895-4351-8f91-10804db684a2 kind: Event lastTimestamp: "2026-08-05T17:37:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:06Z" name: success-200-isvc-8dddc-predictor.18c8f95c284c271a namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: b3c1442c-7234-484f-a7f3-440d3bcbc6c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8dddc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: faefcd91-0895-4351-8f91-10804db684a2 kind: Event lastTimestamp: "2026-08-05T17:37:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:06Z" name: success-200-isvc-8dddc-predictor.18c8f95c28c7ad79 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 1d65e0e4-e8f1-42b7-9ed4-3631dc2e011c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: a6370d7a-b242-4bcf-801a-6766bc3843ac kind: Event lastTimestamp: "2026-08-05T17:37:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:24Z" name: success-200-isvc-8dddc.18c8f9552a990b0e namespace: kserve-ci-e2e-test resourceVersion: "21579" uid: ae081e6a-180f-4e4e-9afa-5d8f85eb3602 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: a6370d7a-b242-4bcf-801a-6766bc3843ac kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-8dddc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8dddc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-8dddc.18c8f9552c0738cf namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: a3ee67c5-cc45-4d0e-8e9d-23f9da93059a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: a6370d7a-b242-4bcf-801a-6766bc3843ac kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8dddc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-8dddc.18c8f9552c077a36 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 1b5c5d23-3b6d-4a29-8d52-21a91f6b401e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8dddc namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: a6370d7a-b242-4bcf-801a-6766bc3843ac kind: Event lastTimestamp: "2026-08-05T17:37:24Z" message: InferenceService [success-200-isvc-8dddc] is Ready metadata: creationTimestamp: "2026-08-05T17:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:24Z" name: success-200-isvc-8dddc.18c8f96042dd0a1b namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: 6b26e6ec-d6a2-43d6-94ec-d1c04fb630ce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:45:51.338265Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-92074-predictor-79d689c86d-w5pjl to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9d65d6f5517 namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: 8b3a4ed8-4082-43ae-8b3f-1e0a77195a3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9d676e96694 namespace: kserve-ci-e2e-test resourceVersion: "25807" uid: 0a29cca3-fc66-4a6a-8f6a-871a6b97f7a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9d6788908f3 namespace: kserve-ci-e2e-test resourceVersion: "25809" uid: 599bf445-183b-4ba2-abe7-d1dabcf98df9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9d67b71071e namespace: kserve-ci-e2e-test resourceVersion: "25811" uid: 5e2f67bc-f2c7-4888-b085-01c44e2dbb17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9d67be029d8 namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: c34a17cf-dd59-413e-b7db-0a72bbf8a282 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9d67beaa083 namespace: kserve-ci-e2e-test resourceVersion: "25814" uid: 1e7db96d-33c9-4878-bbf4-c2b706541f58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9d67ec4f9ae namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: e8c4137a-c60f-4452-8f39-d9fc8005ca6e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9d67f33af94 namespace: kserve-ci-e2e-test resourceVersion: "25816" uid: 7a1108d7-40b1-459c-a967-68ef3b29809d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:46:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:29Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9d6d3d54acd namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: dd051973-44a7-4a56-b15e-d78f70952547 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:01Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9e6b830d689 namespace: kserve-ci-e2e-test resourceVersion: "26624" uid: 9bcaebc1-5bc3-4da0-a0b5-f31451d8348f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:01Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9e6b8322215 namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: 17a71c95-32c1-4214-a80e-be12b90c5108 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92074-predictor-79d689c86d-w5pjl namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 337b1675-3909-4f89-a905-35432d0ff577 kind: Event lastTimestamp: "2026-08-05T17:47:04Z" message: 'Readiness probe failed: Get "https://10.134.0.42:8643/healthz": dial tcp 10.134.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:04Z" name: success-200-isvc-92074-predictor-79d689c86d-w5pjl.18c8f9e75bf4c1cd namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: 6002053d-b2ac-4935-8a4d-e034a6305e05 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-92074-predictor-79d689c86d namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: 7596aa4c-6610-4d17-9271-fe8cc55d340a kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: 'Created pod: success-200-isvc-92074-predictor-79d689c86d-w5pjl' metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor-79d689c86d.18c8f9d65ccbb515 namespace: kserve-ci-e2e-test resourceVersion: "25741" uid: f5541289-afca-44ba-8c9b-93a553b09535 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25712" uid: 6c97d8f6-5c3f-43d4-b7ae-adbca089a418 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: Scaled up replica set success-200-isvc-92074-predictor-79d689c86d from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074-predictor.18c8f9d65844f685 namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 9a76fd26-7ae4-46f1-ac93-0560f925e9c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:46:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: 8944b392-d641-4a25-be6d-d747954cd9b9 kind: Event lastTimestamp: "2026-08-05T17:46:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:21Z" name: success-200-isvc-92074-predictor.18c8f9d9d6d86cd6 namespace: kserve-ci-e2e-test resourceVersion: "26075" uid: 07bd49dc-527f-4830-a805-6f45650b6e25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:46:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: 8944b392-d641-4a25-be6d-d747954cd9b9 kind: Event lastTimestamp: "2026-08-05T17:46:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:21Z" name: success-200-isvc-92074-predictor.18c8f9d9d75eea9d namespace: kserve-ci-e2e-test resourceVersion: "26076" uid: 784a0c57-b031-404a-a5f8-2c987636c0fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:46:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: 8944b392-d641-4a25-be6d-d747954cd9b9 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-92074-predictor.18c8f9e0d5cbbf14 namespace: kserve-ci-e2e-test resourceVersion: "26399" uid: 712c22a2-9e26-41c1-89a2-aec9707aa278 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:46:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: 8944b392-d641-4a25-be6d-d747954cd9b9 kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-92074-predictor.18c8f9e0d64e54b7 namespace: kserve-ci-e2e-test resourceVersion: "26407" uid: 4102ba72-dfe4-41af-8ebe-2b51345424d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92074 namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: 1cb4f61f-f01b-409e-9496-53a397584e28 kind: Event lastTimestamp: "2026-08-05T17:46:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:46:39Z" name: success-200-isvc-92074.18c8f9d6584054c2 namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: e9743f04-8986-48d0-b859-2ae1c557a4b1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92074 namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: 1cb4f61f-f01b-409e-9496-53a397584e28 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-92074": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-92074": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074.18c8f9d66607a466 namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: 7b283818-1400-434f-a182-3e204c9a546d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:45:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92074 namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: 1cb4f61f-f01b-409e-9496-53a397584e28 kind: Event lastTimestamp: "2026-08-05T17:45:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-92074": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:45:51Z" name: success-200-isvc-92074.18c8f9d66607f894 namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: 49756cd1-4df4-438f-a92b-b31dc1c96b31 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92074 namespace: kserve-ci-e2e-test resourceVersion: "26217" uid: 1cb4f61f-f01b-409e-9496-53a397584e28 kind: Event lastTimestamp: "2026-08-05T17:46:39Z" message: InferenceService [success-200-isvc-92074] is Ready metadata: creationTimestamp: "2026-08-05T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:46:39Z" name: success-200-isvc-92074.18c8f9e18fca5b2e namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: 1678b3a5-5212-4186-9bd5-974b2e0c5732 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:26:11.295199Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:26:11Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c39d62c809 namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 2dc11d10-a311-43db-9c83-467e199324d1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a0ee2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c3a80ad6f3 namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 31ecfb9a-a44a-403c-9b99-9fa0bc87b701 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:26:12Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:26:12Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c3db33d973 namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: 0eb391ff-65e3-4df7-93bf-f4cbf5ad28d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:26:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:12Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c3dca70d0d namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 0a58d3ac-c3d8-4ec1-8233-285e28a424ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:26:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:12Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c3dfa9bd7a namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 41a00475-64d9-4e96-abdf-272a46c999d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:26:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:12Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c3e0161bb2 namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: 5f420f1f-9577-4556-b413-f050df8360fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:26:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:12Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c3e01f222e namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: 0a9a4f80-579c-48b0-b87b-cd2c4da37965 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:26:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:12Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c3e417f33b namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: 251392a3-8f69-45eb-a565-833568a29751 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:26:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:12Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c3e4aa1a0e namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: 757447d6-c421-4b26-8841-a389dbdd0b44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:26:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:26:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:49Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f8c41661f818 namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 98b62570-8252-41e0-bb53-97f9b5cff760 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f944cb41608c namespace: kserve-ci-e2e-test resourceVersion: "20180" uid: 60743370-99a0-416c-88e1-45b6a48407c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: cf5fb621-292d-454c-88d6-e66edcd9dfb5 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp.18c8f944cb41d61f namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: e348a28f-14f2-4ba8-ac67-54e04a269bd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a0ee2-predictor-69b645cc54 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 9294dfdc-d9c4-4cad-9299-89732dbba275 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: 'Created pod: success-200-isvc-a0ee2-predictor-69b645cc54-6zzvp' metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:11Z" name: success-200-isvc-a0ee2-predictor-69b645cc54.18c8f8c39cdfc267 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: fe2931e3-3130-4d54-9af8-ab2e79770c97 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: be865896-cb52-49fa-a835-572b78c92847 kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Scaled up replica set success-200-isvc-a0ee2-predictor-69b645cc54 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:11Z" name: success-200-isvc-a0ee2-predictor.18c8f8c39a6d905a namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: dd461468-d1f9-47ee-a2c0-2bcb0f2a30b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 4543dd73-33a6-45a0-a1a1-25e4f7a76c62 kind: Event lastTimestamp: "2026-08-05T17:26:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:41Z" name: success-200-isvc-a0ee2-predictor.18c8f8c71902b197 namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 05149464-6526-4e36-b5b8-ae726ee45604 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 4543dd73-33a6-45a0-a1a1-25e4f7a76c62 kind: Event lastTimestamp: "2026-08-05T17:26:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:41Z" name: success-200-isvc-a0ee2-predictor.18c8f8c71968ccba namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: 4b344a73-2ada-46af-aa63-a086acc7e51c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 4543dd73-33a6-45a0-a1a1-25e4f7a76c62 kind: Event lastTimestamp: "2026-08-05T17:27:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:26Z" name: success-200-isvc-a0ee2-predictor.18c8f8ce1721e9cf namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: cda7f6dd-d8f0-4b7e-8628-d29c35660547 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a0ee2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 4543dd73-33a6-45a0-a1a1-25e4f7a76c62 kind: Event lastTimestamp: "2026-08-05T17:27:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:26Z" name: success-200-isvc-a0ee2-predictor.18c8f8ce1790a453 namespace: kserve-ci-e2e-test resourceVersion: "16759" uid: be0c67e8-e886-4f07-a90a-d421fa300e56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a0ee2 namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: 3e14de15-3df3-43ca-888d-9d6c99dc81fd kind: Event lastTimestamp: "2026-08-05T17:26:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:59Z" name: success-200-isvc-a0ee2.18c8f8c39aa71df3 namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 1349c337-b85e-4212-a80b-760b90121583 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a0ee2 namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: 3e14de15-3df3-43ca-888d-9d6c99dc81fd kind: Event lastTimestamp: "2026-08-05T17:26:59Z" message: InferenceService [success-200-isvc-a0ee2] is Ready metadata: creationTimestamp: "2026-08-05T17:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:59Z" name: success-200-isvc-a0ee2.18c8f8ced1f1728a namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 282fd1f4-5c8f-4d8b-8f74-38eb419a896d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:27:11.247334Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d9755-predictor-d8c999f56-sg8lf to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:27:11Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d192cfac2b namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: bed12f37-cee3-4d09-929c-7f58f3fa1e2b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d9755-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d19ec529d1 namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: a1b47856-0340-4392-b5cd-0fa156980160 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:27:13Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d2058e08bc namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 7e337b80-7728-4203-b4e6-6de2477e67d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d207001af1 namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: fe6a3b9a-e37f-4c7d-84e4-b903f9be3685 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d20a102bfd namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: 669a44b5-5237-4542-b29a-ebfded6fcf16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d20a825257 namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: f98e76cc-6be1-4767-b1ff-f05b3b27b12e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d20a89fc47 namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: 8a4f9cae-2762-4ad9-bf06-b8a11a8955a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d20e4d4f6b namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 4c99f596-59cd-4060-a0b4-99f688874e2b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:27:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:13Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d20ebd0f9e namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: 286f4eeb-88b0-4697-bf92-be1d6b6fe9c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-05T17:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:27:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:50Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f8d21eead0a7 namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: 27a643ba-cc98-4fb4-9719-db13949ab6ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f952c0665557 namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: 42ff4f8a-4749-4490-af8f-407d8ab131ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 37d7cb85-d235-490e-80dc-9e73cc8cfa29 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-d9755-predictor-d8c999f56-sg8lf.18c8f952c066d4de namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: 3e26a6a0-909d-466f-a6bb-3b2132c15025 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d9755-predictor-d8c999f56 namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: 3a73dcc0-2688-47a0-ae62-99e08ad39607 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: 'Created pod: success-200-isvc-d9755-predictor-d8c999f56-sg8lf' metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:11Z" name: success-200-isvc-d9755-predictor-d8c999f56.18c8f8d192329928 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 2e3a42c0-cfda-45b4-83c3-b2f039ad969e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: def04c75-562e-4df0-93d9-4bf09363c391 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Scaled up replica set success-200-isvc-d9755-predictor-d8c999f56 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:11Z" name: success-200-isvc-d9755-predictor.18c8f8d19006d432 namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: 7f5ce016-18a7-44bb-ac1e-897b8e8e0a60 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 0a8acd3e-2857-4042-b2d9-62f2b8b01d10 kind: Event lastTimestamp: "2026-08-05T17:27:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:41Z" name: success-200-isvc-d9755-predictor.18c8f8d50eac9b47 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: cea8f363-3e91-4ea1-845d-ee6e661ae1e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 0a8acd3e-2857-4042-b2d9-62f2b8b01d10 kind: Event lastTimestamp: "2026-08-05T17:27:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:41Z" name: success-200-isvc-d9755-predictor.18c8f8d50f34c36f namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: d584cb00-5f17-4727-87d7-c04bea6dc500 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:27:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 0a8acd3e-2857-4042-b2d9-62f2b8b01d10 kind: Event lastTimestamp: "2026-08-05T17:28:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:28:26Z" name: success-200-isvc-d9755-predictor.18c8f8dc0e36947a namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: 48b30670-991b-4ae1-bb3b-15d2e29b3add reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:27:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d9755-predictor namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 0a8acd3e-2857-4042-b2d9-62f2b8b01d10 kind: Event lastTimestamp: "2026-08-05T17:28:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:28:26Z" name: success-200-isvc-d9755-predictor.18c8f8dc0ea2799f namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 555ba650-b1c7-4356-9414-f902eea05a8a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d9755 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 2bc223ba-14d6-4d91-b859-3430be059010 kind: Event lastTimestamp: "2026-08-05T17:28:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:28:00Z" name: success-200-isvc-d9755.18c8f8d190101c81 namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: 9890e948-371c-4862-9053-12993adf910a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:28:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d9755 namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: 2bc223ba-14d6-4d91-b859-3430be059010 kind: Event lastTimestamp: "2026-08-05T17:28:00Z" message: InferenceService [success-200-isvc-d9755] is Ready metadata: creationTimestamp: "2026-08-05T17:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:28:00Z" name: success-200-isvc-d9755.18c8f8dd167982bb namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 66f314ce-961d-45ff-86b9-3976182a51f0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:47:01.636666Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:47:01Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e6bb8c143e namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 8549e45c-0c1c-4a27-9d4f-683650d7dd97 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-db0ba-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:01Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e6c6bfd8a2 namespace: kserve-ci-e2e-test resourceVersion: "26687" uid: 1db9bd22-7006-4e35-8682-861bd18093e9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:47:02Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e6f8db6887 namespace: kserve-ci-e2e-test resourceVersion: "26703" uid: f8104dc8-2156-4f44-8d7d-459af73c18f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e6fa4159fe namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: 444f22d3-081c-499c-ba63-380c9328062c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e6fe2eb6ef namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 5ab9b635-cac7-4bc8-8288-fd0dd90dd37a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e6fea2c214 namespace: kserve-ci-e2e-test resourceVersion: "26708" uid: 3a6291fb-7e81-44c7-b4c5-2a16ace12189 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e6fead607d namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 746f0ed2-8443-4b83-b429-38832011c2fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e701a9a34a namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 6eb5db98-7c16-4f67-94a3-588da2f77d21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:47:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:02Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e70230605b namespace: kserve-ci-e2e-test resourceVersion: "26713" uid: 010f5517-25b4-4ba9-b282-401af8aad55b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:47:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:47:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:47:39Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8f9e7357109aa namespace: kserve-ci-e2e-test resourceVersion: "27050" uid: 12cdd439-6a69-4fa0-92fe-7069a3395328 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:56:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:56:16Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8fa67e9930a3e namespace: kserve-ci-e2e-test resourceVersion: "30792" uid: b53a10e1-f7e4-4287-8d6e-89fb64486596 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: a5dc3454-c0b1-4150-aa89-3e4034c4e681 kind: Event lastTimestamp: "2026-08-05T17:56:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:56:16Z" name: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6.18c8fa67e99370ed namespace: kserve-ci-e2e-test resourceVersion: "30796" uid: 91280360-30d0-4097-8f0e-e09a19a44705 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-db0ba-predictor-5cd4b48554 namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: e520bc0d-07fb-4b1c-b855-f9464784a6ae kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: 'Created pod: success-200-isvc-db0ba-predictor-5cd4b48554-7xcb6' metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:01Z" name: success-200-isvc-db0ba-predictor-5cd4b48554.18c8f9e6bb09a3df namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: f913f576-7624-4fbd-bc9b-63b804f814ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: c42edd6b-579b-425f-a485-6361098ac19b kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: Scaled up replica set success-200-isvc-db0ba-predictor-5cd4b48554 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:01Z" name: success-200-isvc-db0ba-predictor.18c8f9e6b8fa7c66 namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 7b03e825-ef76-43e0-8dae-50b7e2d19b84 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: 5b2a2119-3923-4392-8587-8717fe5c9afa kind: Event lastTimestamp: "2026-08-05T17:47:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:16Z" name: success-200-isvc-db0ba-predictor.18c8f9ea37ed6706 namespace: kserve-ci-e2e-test resourceVersion: "26857" uid: 89d79b72-9b01-4bbc-afd4-c7369f514204 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: 5b2a2119-3923-4392-8587-8717fe5c9afa kind: Event lastTimestamp: "2026-08-05T17:47:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:47:16Z" name: success-200-isvc-db0ba-predictor.18c8f9ea386f9f39 namespace: kserve-ci-e2e-test resourceVersion: "26859" uid: ff46ff0c-ea5e-4607-a7e9-a2a14760e445 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 5b2a2119-3923-4392-8587-8717fe5c9afa kind: Event lastTimestamp: "2026-08-05T17:48:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:16Z" name: success-200-isvc-db0ba-predictor.18c8f9edb75535d2 namespace: kserve-ci-e2e-test resourceVersion: "27445" uid: abe93b3d-adb9-407d-845a-1f4e5b21f4ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db0ba-predictor namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 5b2a2119-3923-4392-8587-8717fe5c9afa kind: Event lastTimestamp: "2026-08-05T17:48:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:16Z" name: success-200-isvc-db0ba-predictor.18c8f9edb7d315ce namespace: kserve-ci-e2e-test resourceVersion: "27446" uid: d8c7d389-2388-4b0c-97b9-19df54df0872 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db0ba namespace: kserve-ci-e2e-test resourceVersion: "26602" uid: 98b00eea-d47b-45a5-8405-fcccf1b4cf2f kind: Event lastTimestamp: "2026-08-05T17:47:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:49Z" name: success-200-isvc-db0ba.18c8f9e6b9311406 namespace: kserve-ci-e2e-test resourceVersion: "27159" uid: c69ea901-5e07-4156-90cb-bb39eca87083 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db0ba namespace: kserve-ci-e2e-test resourceVersion: "26602" uid: 98b00eea-d47b-45a5-8405-fcccf1b4cf2f kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: 'Failed to update status for InferenceService "success-200-isvc-db0ba": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-db0ba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:01Z" name: success-200-isvc-db0ba.18c8f9e6b9d3c3e0 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: 10b838e1-41fa-46d3-9114-4854fd7be631 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db0ba namespace: kserve-ci-e2e-test resourceVersion: "26602" uid: 98b00eea-d47b-45a5-8405-fcccf1b4cf2f kind: Event lastTimestamp: "2026-08-05T17:47:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-db0ba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:01Z" name: success-200-isvc-db0ba.18c8f9e6b9d4040a namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 844319b7-d671-4f7a-824b-485d0d11d174 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:47:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db0ba namespace: kserve-ci-e2e-test resourceVersion: "27154" uid: 98b00eea-d47b-45a5-8405-fcccf1b4cf2f kind: Event lastTimestamp: "2026-08-05T17:47:49Z" message: InferenceService [success-200-isvc-db0ba] is Ready metadata: creationTimestamp: "2026-08-05T17:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:47:49Z" name: success-200-isvc-db0ba.18c8f9f1f1a30fce namespace: kserve-ci-e2e-test resourceVersion: "27155" uid: 7cbac615-971d-4972-9707-f388fda4390f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:37:36.481608Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21768" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f96325adb85c namespace: kserve-ci-e2e-test resourceVersion: "21776" uid: c4a67dfe-80cb-4025-a49c-f09f1e0240bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e149b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f96331598108 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: 4dd27c84-8e6e-4d4e-b06c-0a270cd73ec2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:37:37Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f96363488177 namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 0d6f23e0-a156-4fc0-ac07-5a166e2c0b89 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f96364d88ef0 namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: e496ea8d-f7d9-4741-b0c5-0a6f5da14323 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f96368b474f5 namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 205d1def-e90f-4c2c-8ecf-f612b9bb3622 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f963692f9c98 namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: 03232d88-f6e1-4ba2-ab4f-d4b8993c2289 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f963693a9115 namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: ae3fcfb4-b669-4604-9127-c3ffe0232763 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f9636d0eb298 namespace: kserve-ci-e2e-test resourceVersion: "21890" uid: 3cced1e6-5ac4-42e5-89c7-0f7eedf0ad51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:37:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:37:37Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f9636d80dd0a namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: 25cf31e3-3daa-4a6b-b2e3-2d752dd03b7c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:38:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:38:15Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f963cb497aaf namespace: kserve-ci-e2e-test resourceVersion: "22213" uid: 51e7b616-ab08-47e0-8d3c-bd9ae29fb24e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f9e44e94e4f1 namespace: kserve-ci-e2e-test resourceVersion: "26350" uid: a0b7dfcc-35d2-48c4-adca-f6cb13ee3d2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 4d68887a-7de6-4f30-a190-e447efb44fbb kind: Event lastTimestamp: "2026-08-05T17:46:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:46:51Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz.18c8f9e44e956f2e namespace: kserve-ci-e2e-test resourceVersion: "26352" uid: de46a9a8-3e20-4519-a92d-c548e761ff9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e149b-predictor-5f5fbfc7cd namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: cb042033-53d0-4d8e-9f5f-63e87098e999 kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'Created pod: success-200-isvc-e149b-predictor-5f5fbfc7cd-vq5fz' metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-e149b-predictor-5f5fbfc7cd.18c8f963253435b8 namespace: kserve-ci-e2e-test resourceVersion: "21772" uid: bd113c9a-0d36-43d5-b09d-1171b9b7cca5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21744" uid: e093ed76-c961-4f58-97ed-28d7cc0bfd5a kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: Scaled up replica set success-200-isvc-e149b-predictor-5f5fbfc7cd from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-e149b-predictor.18c8f96322a36223 namespace: kserve-ci-e2e-test resourceVersion: "21755" uid: bf61309e-f867-4ec3-afdc-6028cbde368d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: 1acafaae-852f-44a3-842c-62d4fb927bc2 kind: Event lastTimestamp: "2026-08-05T17:38:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:51Z" name: success-200-isvc-e149b-predictor.18c8f966a1959fd0 namespace: kserve-ci-e2e-test resourceVersion: "22525" uid: 2e66cead-06b0-472c-89ae-04f2a1c59aaa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-05T17:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: 1acafaae-852f-44a3-842c-62d4fb927bc2 kind: Event lastTimestamp: "2026-08-05T17:38:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:51Z" name: success-200-isvc-e149b-predictor.18c8f966a209db39 namespace: kserve-ci-e2e-test resourceVersion: "22527" uid: cd867344-0931-4745-b47f-70e196049af7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: 1acafaae-852f-44a3-842c-62d4fb927bc2 kind: Event lastTimestamp: "2026-08-05T17:38:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:06Z" name: success-200-isvc-e149b-predictor.18c8f96a20f820ae namespace: kserve-ci-e2e-test resourceVersion: "22141" uid: 493e8db2-bc45-44ab-9a30-0383019f5a16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e149b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: 1acafaae-852f-44a3-842c-62d4fb927bc2 kind: Event lastTimestamp: "2026-08-05T17:38:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:38:06Z" name: success-200-isvc-e149b-predictor.18c8f96a216bb9e6 namespace: kserve-ci-e2e-test resourceVersion: "22143" uid: 25698bce-4152-48ad-b376-620c32d5da66 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e149b namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 77116673-b4fc-4720-a6fb-8931f97e693c kind: Event lastTimestamp: "2026-08-05T17:38:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:38:25Z" name: success-200-isvc-e149b.18c8f96322f15359 namespace: kserve-ci-e2e-test resourceVersion: "22302" uid: 8aff9dce-f095-4f5f-8d01-cfa1161a2865 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e149b namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 77116673-b4fc-4720-a6fb-8931f97e693c kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-e149b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e149b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-e149b.18c8f96323906b80 namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: 38f92c59-43de-4ca1-8eb4-690ae90d15d3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:37:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e149b namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 77116673-b4fc-4720-a6fb-8931f97e693c kind: Event lastTimestamp: "2026-08-05T17:37:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e149b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:37:36Z" name: success-200-isvc-e149b.18c8f9632390cf5d namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: 48baf31a-71e3-450c-9427-2d83371608d1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:38:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e149b namespace: kserve-ci-e2e-test resourceVersion: "22300" uid: 77116673-b4fc-4720-a6fb-8931f97e693c kind: Event lastTimestamp: "2026-08-05T17:38:25Z" message: InferenceService [success-200-isvc-e149b] is Ready metadata: creationTimestamp: "2026-08-05T17:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:38:25Z" name: success-200-isvc-e149b.18c8f96e879f3c9e namespace: kserve-ci-e2e-test resourceVersion: "22301" uid: 33114a7f-36df-4805-8d95-160013590f54 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:35:26.201612Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fba55-predictor-77c4dfc47d-j8skd to ip-10-0-135-2.ec2.internal metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f944d062acaf namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 198ed2c2-ed40-4691-b0b0-3bff2bb3b6ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f944e9c97512 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 52138a8e-4f77-4403-b717-b4993ed67e12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1848" already present on machine metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f944eb59e8c0 namespace: kserve-ci-e2e-test resourceVersion: "20257" uid: 4e40f2a7-d71e-492f-85eb-b6874bbd9dfe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f944ee1a0c4a namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 01bcf09b-6a2d-45e3-8209-9712b1967cb2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f944ee9efab8 namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: d48de861-de65-4d88-804e-8ed1cdc1cfca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f944eea8f822 namespace: kserve-ci-e2e-test resourceVersion: "20260" uid: 11a63738-6ca0-459d-867c-17dfaddb0f7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f944f175399d namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 7c8ea371-68b9-4b0f-bfc3-614cf3326351 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f944f1f6b8b8 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: ba075755-8969-4774-921d-d47e97d36315 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:35:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:36:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:35:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:04Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f94562432701 namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 13ef6640-96c5-4540-b991-95a5c18f1c5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f95529143ce9 namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: c23fd50e-b8cc-4dab-aabd-0a19cb1cd457 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:36:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-05T17:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:36Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f955291747c2 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: e74a5458-66ba-4ec3-b92a-67a17096a4c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: a489ce9e-f83f-47f4-8faf-2358a9349294 kind: Event lastTimestamp: "2026-08-05T17:36:39Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-05T17:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:36:39Z" name: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd.18c8f955eacb73a3 namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: e68001e4-84fb-4a48-a2be-b4ddbeffbd86 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-2.ec2.internal source: component: kubelet host: ip-10-0-135-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fba55-predictor-77c4dfc47d namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: c2b5a339-505b-4c43-a42a-9ae4c8c04fba kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: 'Created pod: success-200-isvc-fba55-predictor-77c4dfc47d-j8skd' metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor-77c4dfc47d.18c8f944cfed33bb namespace: kserve-ci-e2e-test resourceVersion: "20195" uid: 9271500c-4f37-44d4-9618-884b4d5b808c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: ecdfcf10-5d8e-4222-b6b5-05142a7c0578 kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: Scaled up replica set success-200-isvc-fba55-predictor-77c4dfc47d from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55-predictor.18c8f944cbd68465 namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 56665eec-cf9a-45a7-9ffb-865eb8de7f9f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:35:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: dfb50548-5b5c-4153-8bcb-142a79301135 kind: Event lastTimestamp: "2026-08-05T17:35:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:35:56Z" name: success-200-isvc-fba55-predictor.18c8f9484b13dfb1 namespace: kserve-ci-e2e-test resourceVersion: "20512" uid: c349ad81-3663-4ebb-95b9-e3bc1ea992d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:35:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: dfb50548-5b5c-4153-8bcb-142a79301135 kind: Event lastTimestamp: "2026-08-05T17:35:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:35:56Z" name: success-200-isvc-fba55-predictor.18c8f9484b93dfdf namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: 3128f571-9c80-4266-af41-4457a265928b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: dfb50548-5b5c-4153-8bcb-142a79301135 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-fba55-predictor.18c8f94f49c9d94e namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: 3454a9d6-f6be-4407-acda-45cbc3bba36a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fba55-predictor namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: dfb50548-5b5c-4153-8bcb-142a79301135 kind: Event lastTimestamp: "2026-08-05T17:36:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-05T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:36:26Z" name: success-200-isvc-fba55-predictor.18c8f94f4a429610 namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 0d606c05-cb77-4680-9089-1cf08bdbe8c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fba55 namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 78903ba8-e9dc-40db-9738-3d877bfc61df kind: Event lastTimestamp: "2026-08-05T17:36:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:14Z" name: success-200-isvc-fba55.18c8f944cbd9fe53 namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 17e69c39-3136-4d74-b84f-c33cd941ec91 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fba55 namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 78903ba8-e9dc-40db-9738-3d877bfc61df kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-fba55": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fba55": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55.18c8f944cca30aca namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: ffe161a8-fe1e-4fa8-ae1a-2195c779cb76 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fba55 namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 78903ba8-e9dc-40db-9738-3d877bfc61df kind: Event lastTimestamp: "2026-08-05T17:35:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fba55": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-05T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:35:26Z" name: success-200-isvc-fba55.18c8f944cca35bce namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 74c20af2-43a6-4a14-bf11-c869fdbb28a5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:36:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fba55 namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 78903ba8-e9dc-40db-9738-3d877bfc61df kind: Event lastTimestamp: "2026-08-05T17:36:14Z" message: InferenceService [success-200-isvc-fba55] is Ready metadata: creationTimestamp: "2026-08-05T17:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:36:14Z" name: success-200-isvc-fba55.18c8f95020639036 namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 37a42390-5c12-4ec0-8da4-529885641f9c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:48:01.406691Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-741a0-7dd5d757cf-bthq5 namespace: kserve-ci-e2e-test resourceVersion: "27255" uid: eb0a6db1-307f-486e-acd8-ac0ff42fae2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-741a0-7dd5d757cf-bthq5 to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:48:01Z" name: switch-graph-741a0-7dd5d757cf-bthq5.18c8f9f4a61dac9b namespace: kserve-ci-e2e-test resourceVersion: "27261" uid: 395238a2-1926-4355-bc55-23388c7f7a5e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-741a0-7dd5d757cf-bthq5 namespace: kserve-ci-e2e-test resourceVersion: "27266" uid: eb0a6db1-307f-486e-acd8-ac0ff42fae2f kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:48:01Z" name: switch-graph-741a0-7dd5d757cf-bthq5.18c8f9f4c03fb061 namespace: kserve-ci-e2e-test resourceVersion: "27306" uid: f0ede0cc-b577-4eab-9ad4-8ddc9c6cc621 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-741a0} kind: Pod name: switch-graph-741a0-7dd5d757cf-bthq5 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: eb0a6db1-307f-486e-acd8-ac0ff42fae2f kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:48:01Z" name: switch-graph-741a0-7dd5d757cf-bthq5.18c8f9f4c22a4047 namespace: kserve-ci-e2e-test resourceVersion: "27310" uid: dfd948bb-7c79-407f-988f-62a5204b4785 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-741a0} kind: Pod name: switch-graph-741a0-7dd5d757cf-bthq5 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: eb0a6db1-307f-486e-acd8-ac0ff42fae2f kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: 'Created container: switch-graph-741a0' metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:48:01Z" name: switch-graph-741a0-7dd5d757cf-bthq5.18c8f9f4c51ebac7 namespace: kserve-ci-e2e-test resourceVersion: "27312" uid: 28b53148-6eee-4819-af46-aab43f619e9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-741a0} kind: Pod name: switch-graph-741a0-7dd5d757cf-bthq5 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: eb0a6db1-307f-486e-acd8-ac0ff42fae2f kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: Started container switch-graph-741a0 metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:48:01Z" name: switch-graph-741a0-7dd5d757cf-bthq5.18c8f9f4c5a7a031 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: ad2d2a3d-c956-41f9-b916-58831b124d9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T18:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-741a0} kind: Pod name: switch-graph-741a0-7dd5d757cf-bthq5 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: eb0a6db1-307f-486e-acd8-ac0ff42fae2f kind: Event lastTimestamp: "2026-08-05T18:04:20Z" message: Stopping container switch-graph-741a0 metadata: creationTimestamp: "2026-08-05T18:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T18:04:20Z" name: switch-graph-741a0-7dd5d757cf-bthq5.18c8fad89eeccdb5 namespace: kserve-ci-e2e-test resourceVersion: "34131" uid: f3a040f1-3e8d-45d7-abb0-05e919e14545 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-05T18:04:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-741a0} kind: Pod name: switch-graph-741a0-7dd5d757cf-bthq5 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: eb0a6db1-307f-486e-acd8-ac0ff42fae2f kind: Event lastTimestamp: "2026-08-05T18:04:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T18:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T18:04:43Z" name: switch-graph-741a0-7dd5d757cf-bthq5.18c8fad964ce93f2 namespace: kserve-ci-e2e-test resourceVersion: "34377" uid: fc1b27cd-b988-4df8-8b28-de6ded6c367b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-741a0-7dd5d757cf namespace: kserve-ci-e2e-test resourceVersion: "27252" uid: 007d827d-7ab9-48de-9ce2-84ef1f6470f0 kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: 'Created pod: switch-graph-741a0-7dd5d757cf-bthq5' metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:01Z" name: switch-graph-741a0-7dd5d757cf.18c8f9f4a56a9980 namespace: kserve-ci-e2e-test resourceVersion: "27259" uid: 7d069c9b-122b-4248-a469-b137633a5dfd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-741a0 namespace: kserve-ci-e2e-test resourceVersion: "27250" uid: a10015cb-8cc9-4bbc-b0f1-a9a112f78edf kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: Scaled up replica set switch-graph-741a0-7dd5d757cf from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:01Z" name: switch-graph-741a0.18c8f9f4a37b2d88 namespace: kserve-ci-e2e-test resourceVersion: "27253" uid: 99833012-390c-4dcf-ba5b-ed8afe407604 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:48:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-741a0 namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: 801a1268-f212-4a0f-b406-65d337c3ab8f kind: Event lastTimestamp: "2026-08-05T17:48:01Z" message: InferenceGraph [switch-graph-741a0] is Ready metadata: creationTimestamp: "2026-08-05T17:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:48:01Z" name: switch-graph-741a0.18c8f9f4ac667504 namespace: kserve-ci-e2e-test resourceVersion: "27270" uid: 724821ae-3904-48a3-9062-13bd76c6cfd5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:48:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-741a0 namespace: kserve-ci-e2e-test resourceVersion: "27251" uid: 8d727710-de21-4226-91c1-8eb289b336f3 kind: Event lastTimestamp: "2026-08-05T17:48:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:46Z" name: switch-graph-741a0.18c8f9f82269602a namespace: kserve-ci-e2e-test resourceVersion: "27658" uid: 06e14d6f-151a-4592-825f-6ef0025b76b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-05T17:48:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-741a0 namespace: kserve-ci-e2e-test resourceVersion: "27251" uid: 8d727710-de21-4226-91c1-8eb289b336f3 kind: Event lastTimestamp: "2026-08-05T17:48:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:48:46Z" name: switch-graph-741a0.18c8f9f822bb594c namespace: kserve-ci-e2e-test resourceVersion: "27660" uid: 4574635d-1816-4333-a1c5-1ba80b737065 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:26:00.896704Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-829a9-54c877bfc8-tsdm4 namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: 62a317fa-a65b-4257-b262-b2bc5c93945a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-829a9-54c877bfc8-tsdm4 to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:26:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:26:00Z" name: switch-graph-829a9-54c877bfc8-tsdm4.18c8f8c131964c11 namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: ed536838-3603-453f-8f94-3d914a3b3379 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-829a9-54c877bfc8-tsdm4 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 62a317fa-a65b-4257-b262-b2bc5c93945a kind: Event lastTimestamp: "2026-08-05T17:26:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-829a9-serving-cert" not found' metadata: creationTimestamp: "2026-08-05T17:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:01Z" name: switch-graph-829a9-54c877bfc8-tsdm4.18c8f8c138abef8f namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 2939e6d8-4149-4f61-94ba-e33ec42e5b93 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-829a9-54c877bfc8-tsdm4 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 62a317fa-a65b-4257-b262-b2bc5c93945a kind: Event lastTimestamp: "2026-08-05T17:26:01Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:26:01Z" name: switch-graph-829a9-54c877bfc8-tsdm4.18c8f8c16ed67061 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: f0358e19-2c5c-4bf6-bcac-9c853ad5cd70 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-829a9} kind: Pod name: switch-graph-829a9-54c877bfc8-tsdm4 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 62a317fa-a65b-4257-b262-b2bc5c93945a kind: Event lastTimestamp: "2026-08-05T17:26:01Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" metadata: creationTimestamp: "2026-08-05T17:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:01Z" name: switch-graph-829a9-54c877bfc8-tsdm4.18c8f8c170136df6 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 33e2d286-c555-486c-971d-69237fdd8f68 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-829a9} kind: Pod name: switch-graph-829a9-54c877bfc8-tsdm4 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 62a317fa-a65b-4257-b262-b2bc5c93945a kind: Event lastTimestamp: "2026-08-05T17:26:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" in 2.315s (2.315s including waiting). Image size: 218259558 bytes.' metadata: creationTimestamp: "2026-08-05T17:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:04Z" name: switch-graph-829a9-54c877bfc8-tsdm4.18c8f8c1fa1a7f1b namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: ed906375-8af5-4cb7-9147-54b76871246c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-829a9} kind: Pod name: switch-graph-829a9-54c877bfc8-tsdm4 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 62a317fa-a65b-4257-b262-b2bc5c93945a kind: Event lastTimestamp: "2026-08-05T17:26:04Z" message: 'Created container: switch-graph-829a9' metadata: creationTimestamp: "2026-08-05T17:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:04Z" name: switch-graph-829a9-54c877bfc8-tsdm4.18c8f8c1fcf11094 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 405b0b78-63d5-44f2-90b0-1625a1304455 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-829a9} kind: Pod name: switch-graph-829a9-54c877bfc8-tsdm4 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 62a317fa-a65b-4257-b262-b2bc5c93945a kind: Event lastTimestamp: "2026-08-05T17:26:04Z" message: Started container switch-graph-829a9 metadata: creationTimestamp: "2026-08-05T17:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:04Z" name: switch-graph-829a9-54c877bfc8-tsdm4.18c8f8c1fd653562 namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: a1a023b5-125c-4d39-a76c-fe3e3344e0fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-829a9} kind: Pod name: switch-graph-829a9-54c877bfc8-tsdm4 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 62a317fa-a65b-4257-b262-b2bc5c93945a kind: Event lastTimestamp: "2026-08-05T17:26:11Z" message: Stopping container switch-graph-829a9 metadata: creationTimestamp: "2026-08-05T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:11Z" name: switch-graph-829a9-54c877bfc8-tsdm4.18c8f8c391d262f6 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 09b4cf26-8a65-4c67-9ba2-11800c54ff59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:26:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-829a9} kind: Pod name: switch-graph-829a9-54c877bfc8-tsdm4 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 62a317fa-a65b-4257-b262-b2bc5c93945a kind: Event lastTimestamp: "2026-08-05T17:26:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:26:40Z" name: switch-graph-829a9-54c877bfc8-tsdm4.18c8f8c49687d3b8 namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: 3794de5b-dac1-4f07-a367-3fa8156e31d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-829a9-54c877bfc8 namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: 7ee73525-da59-4e19-81ef-e2de910a732b kind: Event lastTimestamp: "2026-08-05T17:26:00Z" message: 'Created pod: switch-graph-829a9-54c877bfc8-tsdm4' metadata: creationTimestamp: "2026-08-05T17:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:00Z" name: switch-graph-829a9-54c877bfc8.18c8f8c13116bc19 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 72cedca8-c883-419c-a79a-77ad858f58f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-829a9 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: f493809b-89ce-49e2-a8cf-a03048349ba1 kind: Event lastTimestamp: "2026-08-05T17:26:00Z" message: Scaled up replica set switch-graph-829a9-54c877bfc8 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:26:00Z" name: switch-graph-829a9.18c8f8c1301b894d namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: 30d8605d-98da-4823-9186-080481eb869d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:26:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-829a9 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: e73b267a-0426-4d03-9509-fab4c7176ffb kind: Event lastTimestamp: "2026-08-05T17:26:00Z" message: InferenceGraph [switch-graph-829a9] is Ready metadata: creationTimestamp: "2026-08-05T17:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:26:00Z" name: switch-graph-829a9.18c8f8c13457ce3f namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: bcb84841-e7d1-4bc1-b3e7-74ca400eaa29 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-05T17:27:11.370364Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a0ee2-54954f7959-hxcg6 namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: 72df30ba-8071-45e0-98cc-da8ff0a18f61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-a0ee2-54954f7959-hxcg6 to ip-10-0-137-104.ec2.internal metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-05T17:27:11Z" name: switch-graph-a0ee2-54954f7959-hxcg6.18c8f8d19a24f7d5 namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: a150b1a7-68ef-4462-a940-e3d8332ff449 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76b448779b-6t5gp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a0ee2-54954f7959-hxcg6 namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: 72df30ba-8071-45e0-98cc-da8ff0a18f61 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-05T17:27:11Z" name: switch-graph-a0ee2-54954f7959-hxcg6.18c8f8d1b3bb2777 namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: d7288c25-89f1-47c4-a4a1-5575d556d10d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0ee2} kind: Pod name: switch-graph-a0ee2-54954f7959-hxcg6 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 72df30ba-8071-45e0-98cc-da8ff0a18f61 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ce7c3058e7df0b4a1b07c83c58a6293353094b90f1440f7751ce51d2cb11c8c" already present on machine metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: switch-graph-a0ee2-54954f7959-hxcg6.18c8f8d1b5a5ae68 namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 402eb672-8c1a-4151-a31a-78e7c3f7cb89 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0ee2} kind: Pod name: switch-graph-a0ee2-54954f7959-hxcg6 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 72df30ba-8071-45e0-98cc-da8ff0a18f61 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: 'Created container: switch-graph-a0ee2' metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: switch-graph-a0ee2-54954f7959-hxcg6.18c8f8d1b8aa6c1e namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 4ed71094-c3f9-4e57-9170-fa5e7e54226e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0ee2} kind: Pod name: switch-graph-a0ee2-54954f7959-hxcg6 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 72df30ba-8071-45e0-98cc-da8ff0a18f61 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Started container switch-graph-a0ee2 metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:27:11Z" name: switch-graph-a0ee2-54954f7959-hxcg6.18c8f8d1b90c9beb namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 39596598-f109-4dda-8ab4-4367105923a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:35:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0ee2} kind: Pod name: switch-graph-a0ee2-54954f7959-hxcg6 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 72df30ba-8071-45e0-98cc-da8ff0a18f61 kind: Event lastTimestamp: "2026-08-05T17:35:25Z" message: Stopping container switch-graph-a0ee2 metadata: creationTimestamp: "2026-08-05T17:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:25Z" name: switch-graph-a0ee2-54954f7959-hxcg6.18c8f944c273ad20 namespace: kserve-ci-e2e-test resourceVersion: "20153" uid: 8dfae1c7-25c7-41fd-a70f-eccb0b971169 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-05T17:35:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0ee2} kind: Pod name: switch-graph-a0ee2-54954f7959-hxcg6 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 72df30ba-8071-45e0-98cc-da8ff0a18f61 kind: Event lastTimestamp: "2026-08-05T17:35:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-05T17:35:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-05T17:35:53Z" name: switch-graph-a0ee2-54954f7959-hxcg6.18c8f94568aabf7a namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: 1a180625-cb9d-4d51-8797-58f18ca80b23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-104.ec2.internal source: component: kubelet host: ip-10-0-137-104.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-a0ee2-54954f7959 namespace: kserve-ci-e2e-test resourceVersion: "16538" uid: d0f455ea-03cc-469d-a867-08dddd62e96c kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: 'Created pod: switch-graph-a0ee2-54954f7959-hxcg6' metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:11Z" name: switch-graph-a0ee2-54954f7959.18c8f8d1998226cb namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 8a174500-5b51-40ad-9111-0a591eb64f62 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-a0ee2 namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 3dbd85e3-4945-4cab-bf76-b6d401a32598 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: Scaled up replica set switch-graph-a0ee2-54954f7959 from 0 to 1 metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:11Z" name: switch-graph-a0ee2.18c8f8d198a0048d namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 337342a2-861b-4e2e-871a-8546f9c00d8f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-05T17:27:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-a0ee2 namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 5af99c69-317b-4a77-81c1-821b9d843a37 kind: Event lastTimestamp: "2026-08-05T17:27:11Z" message: InferenceGraph [switch-graph-a0ee2] is Ready metadata: creationTimestamp: "2026-08-05T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-05T17:27:11Z" name: switch-graph-a0ee2.18c8f8d19e7ccdef namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: bd8b76e6-214a-4b25-bd6d-e81c91a7d998 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a0ee2 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 6725f8e7-4c1c-486a-b167-af376d9ab488 kind: Event lastTimestamp: "2026-08-05T17:27:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:41Z" name: switch-graph-a0ee2.18c8f8d51744fa0e namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 6da3d846-516c-4ea1-9021-d3247f5fddf2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-05T17:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a0ee2 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 6725f8e7-4c1c-486a-b167-af376d9ab488 kind: Event lastTimestamp: "2026-08-05T17:27:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-05T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-05T17:27:41Z" name: switch-graph-a0ee2.18c8f8d517b59189 namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: b000de53-084a-4d42-8ba2-ba8fe4350e12 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34397"