--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:42:35.174403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-87dcc-698df6b5bc-tdg46 namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: 31644f98-10f7-469a-aec0-be065f15beaa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-87dcc-698df6b5bc-tdg46 to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:42:35Z" name: ensemble-graph-87dcc-698df6b5bc-tdg46.18be9706278c79ba namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: f2fbdbe5-e434-4609-a256-df805a1f9891 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-87dcc-698df6b5bc-tdg46 namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: 31644f98-10f7-469a-aec0-be065f15beaa kind: Event lastTimestamp: "2026-07-02T21:42:35Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:42:35Z" name: ensemble-graph-87dcc-698df6b5bc-tdg46.18be970640f93859 namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: 95053d44-a795-4789-a3c2-bba0fe38cb1d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-87dcc} kind: Pod name: ensemble-graph-87dcc-698df6b5bc-tdg46 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 31644f98-10f7-469a-aec0-be065f15beaa kind: Event lastTimestamp: "2026-07-02T21:42:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:35Z" name: ensemble-graph-87dcc-698df6b5bc-tdg46.18be97064228a432 namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: 9cfbb5c8-2268-4de7-a3bb-8f34557b858b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-87dcc} kind: Pod name: ensemble-graph-87dcc-698df6b5bc-tdg46 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 31644f98-10f7-469a-aec0-be065f15beaa kind: Event lastTimestamp: "2026-07-02T21:42:35Z" message: 'Created container: ensemble-graph-87dcc' metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:35Z" name: ensemble-graph-87dcc-698df6b5bc-tdg46.18be970644e77dc6 namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: e54f9ec9-ac18-4e2b-825a-b2d7294849ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-87dcc} kind: Pod name: ensemble-graph-87dcc-698df6b5bc-tdg46 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 31644f98-10f7-469a-aec0-be065f15beaa kind: Event lastTimestamp: "2026-07-02T21:42:35Z" message: Started container ensemble-graph-87dcc metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:35Z" name: ensemble-graph-87dcc-698df6b5bc-tdg46.18be970645531ed4 namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: 079cb5ab-c21e-4dc5-89c1-6edd32e2f00e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-87dcc} kind: Pod name: ensemble-graph-87dcc-698df6b5bc-tdg46 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 31644f98-10f7-469a-aec0-be065f15beaa kind: Event lastTimestamp: "2026-07-02T21:50:49Z" message: Stopping container ensemble-graph-87dcc metadata: creationTimestamp: "2026-07-02T21:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:49Z" name: ensemble-graph-87dcc-698df6b5bc-tdg46.18be977952d63815 namespace: kserve-ci-e2e-test resourceVersion: "25591" uid: 42705173-a67c-4820-bbc9-75746760383d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-87dcc} kind: Pod name: ensemble-graph-87dcc-698df6b5bc-tdg46 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 31644f98-10f7-469a-aec0-be065f15beaa kind: Event lastTimestamp: "2026-07-02T21:51:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: ensemble-graph-87dcc-698df6b5bc-tdg46.18be9779d49f2f56 namespace: kserve-ci-e2e-test resourceVersion: "26047" uid: b70e541f-3bbe-4af6-96bb-90eb43394c50 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-87dcc-698df6b5bc namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: 52723f49-347d-455d-bdfc-922b69c044c2 kind: Event lastTimestamp: "2026-07-02T21:42:35Z" message: 'Created pod: ensemble-graph-87dcc-698df6b5bc-tdg46' metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:35Z" name: ensemble-graph-87dcc-698df6b5bc.18be970626edff63 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: e5bbd5af-5a30-48a8-9cc1-0aa0cc419bcf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-87dcc namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: ce76e1f5-c0ec-4e0a-b83f-dac868e97a91 kind: Event lastTimestamp: "2026-07-02T21:42:35Z" message: Scaled up replica set ensemble-graph-87dcc-698df6b5bc from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:35Z" name: ensemble-graph-87dcc.18be970625dda545 namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: c6039a73-0b44-43e2-8635-041e0364ace2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-87dcc namespace: kserve-ci-e2e-test resourceVersion: "21836" uid: 082be45f-9e68-4860-a083-f0c8c1a0c603 kind: Event lastTimestamp: "2026-07-02T21:42:35Z" message: InferenceGraph [ensemble-graph-87dcc] is Ready metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:35Z" name: ensemble-graph-87dcc.18be97062b320c79 namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: ab35690d-9f30-438c-b3ec-023c12d67b75 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-87dcc namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 43349330-ee6e-4a02-882e-c1c25ee95c20 kind: Event lastTimestamp: "2026-07-02T21:43:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:05Z" name: ensemble-graph-87dcc.18be9709a437cf00 namespace: kserve-ci-e2e-test resourceVersion: "22136" uid: 9796ac06-227c-4571-9341-c5189956896d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-87dcc namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 43349330-ee6e-4a02-882e-c1c25ee95c20 kind: Event lastTimestamp: "2026-07-02T21:43:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:05Z" name: ensemble-graph-87dcc.18be9709a494145d namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: fa804557-90e8-44fa-b17b-a1c62661004b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:41:24.865424Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ee70d-579c96c64c-pwwnz namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: 924eaf82-9435-478a-8080-cb22c6ce524e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-ee70d-579c96c64c-pwwnz to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:41:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:41:24Z" name: ensemble-graph-ee70d-579c96c64c-pwwnz.18be96f5c8ce98c9 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 3eed1c0f-de9b-4277-b93a-424c267671b4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:25Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ee70d-579c96c64c-pwwnz namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: 924eaf82-9435-478a-8080-cb22c6ce524e kind: Event lastTimestamp: "2026-07-02T21:41:25Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:41:25Z" name: ensemble-graph-ee70d-579c96c64c-pwwnz.18be96f5e21841e6 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 422dc2f5-efed-45e6-9c70-e94fca6ca2f3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ee70d} kind: Pod name: ensemble-graph-ee70d-579c96c64c-pwwnz namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 924eaf82-9435-478a-8080-cb22c6ce524e kind: Event lastTimestamp: "2026-07-02T21:41:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:25Z" name: ensemble-graph-ee70d-579c96c64c-pwwnz.18be96f5e4857697 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 7de600a0-7860-4b3a-9244-f7119feb0a21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ee70d} kind: Pod name: ensemble-graph-ee70d-579c96c64c-pwwnz namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 924eaf82-9435-478a-8080-cb22c6ce524e kind: Event lastTimestamp: "2026-07-02T21:41:25Z" message: 'Created container: ensemble-graph-ee70d' metadata: creationTimestamp: "2026-07-02T21:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:25Z" name: ensemble-graph-ee70d-579c96c64c-pwwnz.18be96f5e7640845 namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: 01f562fa-e72c-4bf3-b984-f5b6820db3ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ee70d} kind: Pod name: ensemble-graph-ee70d-579c96c64c-pwwnz namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 924eaf82-9435-478a-8080-cb22c6ce524e kind: Event lastTimestamp: "2026-07-02T21:41:25Z" message: Started container ensemble-graph-ee70d metadata: creationTimestamp: "2026-07-02T21:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:25Z" name: ensemble-graph-ee70d-579c96c64c-pwwnz.18be96f5e7ceff25 namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 32fffa0b-af86-4ffd-abfe-bcb26013741c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ee70d} kind: Pod name: ensemble-graph-ee70d-579c96c64c-pwwnz namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 924eaf82-9435-478a-8080-cb22c6ce524e kind: Event lastTimestamp: "2026-07-02T21:41:34Z" message: Stopping container ensemble-graph-ee70d metadata: creationTimestamp: "2026-07-02T21:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:34Z" name: ensemble-graph-ee70d-579c96c64c-pwwnz.18be96f821011b53 namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: f3f05dc2-9b92-44e8-afcb-790584ac6815 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:41:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ee70d} kind: Pod name: ensemble-graph-ee70d-579c96c64c-pwwnz namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 924eaf82-9435-478a-8080-cb22c6ce524e kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:41:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: ensemble-graph-ee70d-579c96c64c-pwwnz.18be96f88c436331 namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: cc510473-ebaa-47a4-bdba-55191494a79f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-ee70d-579c96c64c namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 35d25dc7-a3d5-406b-b2f5-6a42abf4ad75 kind: Event lastTimestamp: "2026-07-02T21:41:24Z" message: 'Created pod: ensemble-graph-ee70d-579c96c64c-pwwnz' metadata: creationTimestamp: "2026-07-02T21:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:24Z" name: ensemble-graph-ee70d-579c96c64c.18be96f5c86b0103 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: e9131acd-2d3c-4adb-aaf3-5bb07dd30c75 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-ee70d namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 685b0811-6b99-4d91-88af-8426cf9878bb kind: Event lastTimestamp: "2026-07-02T21:41:24Z" message: Scaled up replica set ensemble-graph-ee70d-579c96c64c from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:24Z" name: ensemble-graph-ee70d.18be96f5c77bd780 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 81427c45-69ff-4701-b5cd-58ee3869b1d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ee70d namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: d7232b13-e11b-4863-8b4a-924ecb717bab kind: Event lastTimestamp: "2026-07-02T21:41:24Z" message: InferenceGraph [ensemble-graph-ee70d] is Ready metadata: creationTimestamp: "2026-07-02T21:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:24Z" name: ensemble-graph-ee70d.18be96f5ccbfbc6f namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: d834d40a-01f7-41a7-b0d9-9b28532c68da reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:40:50.891779Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1702f-predictor-786664c4bf-btbvk to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:40:50Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96eddfd1df09 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 916452fb-4300-4f59-b339-1fa564f8f3ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1702f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96edef8a0c31 namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: 756285ef-dcd7-4a4f-a8af-434d8482dfdb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:40:51Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96ee1dc99ccc namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: 6aaa4b25-b461-4d66-8ad3-7feac6db9091 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96ee1f5931fe namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 1505a659-dbaa-4619-84e2-ca54540557a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:40:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:52Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96ee227e18a1 namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 9073c2d7-6b18-4c01-81ec-673222ae6dbe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:40:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:52Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96ee22ea6676 namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: fa2b34d9-380c-41e8-be52-b2a03b898068 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:40:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:52Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96ee22f42f3e namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: e64a3193-a511-4a6d-a239-4c8461f8f86f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:40:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:52Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96ee276d1b94 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 546d2386-f7ce-4a1a-a533-ae48266c7c22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:40:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:52Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96ee27e70048 namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: 3a3448ab-07fb-41ca-b599-b3f3cb525853 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:41:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:28Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96ee337e820b namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: dceb0600-29ae-4174-8ff8-8dea08a53854 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96fe37c67f4a namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: be17478a-f764-4ca4-bd72-5b3604ca1e90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96fe37cab880 namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 7f7f7c80-aca3-4e5c-9498-0dd6ed232589 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1702f-predictor-786664c4bf-btbvk namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c0530481-196f-444c-ae68-287e6d9b4138 kind: Event lastTimestamp: "2026-07-02T21:42:03Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:03Z" name: error-404-isvc-1702f-predictor-786664c4bf-btbvk.18be96febba1a34c namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 8b03afaf-6fd8-490c-b0e7-f6726d433109 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1702f-predictor-786664c4bf namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: e82d76f3-e69d-462c-aea4-0fe38f43b496 kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: 'Created pod: error-404-isvc-1702f-predictor-786664c4bf-btbvk' metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:50Z" name: error-404-isvc-1702f-predictor-786664c4bf.18be96eddf55db9d namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: 1abf1e72-0e11-4fc4-909c-c88297defafc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: 6cb0e492-8cb0-4e24-a23d-f81d6c06b772 kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: Scaled up replica set error-404-isvc-1702f-predictor-786664c4bf from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:50Z" name: error-404-isvc-1702f-predictor.18be96eddda1e8b4 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 9ba2cb80-6b0d-44f5-a2cb-53d7d3cd39cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:41:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: f3805cda-63e8-4837-96cf-ce1a8fb75df0 kind: Event lastTimestamp: "2026-07-02T21:41:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:20Z" name: error-404-isvc-1702f-predictor.18be96f15bfe5382 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 3e364191-8598-4f00-88a9-44b685794310 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:41:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: f3805cda-63e8-4837-96cf-ce1a8fb75df0 kind: Event lastTimestamp: "2026-07-02T21:41:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:20Z" name: error-404-isvc-1702f-predictor.18be96f15c785be1 namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: a166409c-81b0-4e53-a980-ee0fc609d576 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: f3805cda-63e8-4837-96cf-ce1a8fb75df0 kind: Event lastTimestamp: "2026-07-02T21:41:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:50Z" name: error-404-isvc-1702f-predictor.18be96f859d22e10 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: 70146a10-b07d-443c-98b2-cc3337587dfa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: f3805cda-63e8-4837-96cf-ce1a8fb75df0 kind: Event lastTimestamp: "2026-07-02T21:41:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:50Z" name: error-404-isvc-1702f-predictor.18be96f85a33d5a7 namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: 426b4a87-3111-4900-85ad-d8493533ee16 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1702f namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 9ab2cd3f-2526-437e-839f-1a2665f5637c kind: Event lastTimestamp: "2026-07-02T21:41:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:38Z" name: error-404-isvc-1702f.18be96eddda40ce7 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 2283cd7a-1000-461b-8815-5c9673641644 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1702f namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 9ab2cd3f-2526-437e-839f-1a2665f5637c kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-1702f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1702f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:50Z" name: error-404-isvc-1702f.18be96edde432480 namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: 40677827-336b-4e56-a548-40145780a426 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1702f namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 9ab2cd3f-2526-437e-839f-1a2665f5637c kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1702f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:50Z" name: error-404-isvc-1702f.18be96edde437153 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 5ae3d1a0-757e-4d0e-ac2b-bde890a61fda reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1702f namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 9ab2cd3f-2526-437e-839f-1a2665f5637c kind: Event lastTimestamp: "2026-07-02T21:41:38Z" message: InferenceService [error-404-isvc-1702f] is Ready metadata: creationTimestamp: "2026-07-02T21:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:38Z" name: error-404-isvc-1702f.18be96f8ee3e66cc namespace: kserve-ci-e2e-test resourceVersion: "21144" uid: 1792bc6e-5821-4b46-a50a-bfae9a036ee3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:52:00.367373Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26581" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be9789bfacc002 namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: 5a73adbf-ad8d-4086-8e1e-4e9db366e09e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26592" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be9789d97e7bfd namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: c6e8b7c5-83b6-4002-ac98-8aa4b49b6773 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be9789dac0635f namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: d409704b-c3d6-4271-91db-0ca58d563398 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be9789ddafa0d7 namespace: kserve-ci-e2e-test resourceVersion: "26612" uid: 45006209-76c9-498e-82b9-7af429a6e45e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be9789de220638 namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: d8de5fef-3acd-409f-87c3-ca86fac9bdc4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be9789de2b2911 namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: dfc5ff31-4cf6-40f2-8b01-730141071293 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be9789e0f5c19a namespace: kserve-ci-e2e-test resourceVersion: "26617" uid: 2ab4c380-767d-41cf-86fc-4d0e6af1da76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be9789e167763f namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: 8d825f7a-18e0-4a92-8325-576c362413ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:52:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T21:52:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:52:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:38Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be978a2e30b206 namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: 6f20fff0-986d-4619-886b-9f73fe663756 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T22:01:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T22:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:01:15Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be980ae9e36c66 namespace: kserve-ci-e2e-test resourceVersion: "30669" uid: 32850063-4c46-4e01-b1a6-687667d420d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T22:01:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T22:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:01:15Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be980ae9e40763 namespace: kserve-ci-e2e-test resourceVersion: "30670" uid: 324a0850-8693-4989-8e30-45e3c4feab27 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:01:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T22:01:19Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-02T22:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:01:19Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be980bde4ccd0b namespace: kserve-ci-e2e-test resourceVersion: "30705" uid: 619d9803-3fcb-4fa3-9c5e-420964b13fa9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:01:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 4af0d93e-02b1-4c46-832a-6580f162dfbb kind: Event lastTimestamp: "2026-07-02T22:01:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: i/o timeout' metadata: creationTimestamp: "2026-07-02T22:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:01:19Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq.18be980bde9bdee9 namespace: kserve-ci-e2e-test resourceVersion: "30706" uid: 374c0fad-970a-48ab-9eab-d161082fa21e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-67ac0-predictor-6479ffbcb7 namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: ebcacfce-4bf3-421d-b928-ed93eb09d2d7 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: 'Created pod: error-404-isvc-67ac0-predictor-6479ffbcb7-wlhcq' metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor-6479ffbcb7.18be9789bf221a4c namespace: kserve-ci-e2e-test resourceVersion: "26582" uid: f2bc7343-511d-4400-85f8-cc7bbd7cbcca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: fa95df0a-ebe6-49ab-bd9f-f92c055dc571 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Scaled up replica set error-404-isvc-67ac0-predictor-6479ffbcb7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0-predictor.18be9789bd9f53ad namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 5472b8f3-1cec-4c22-92e1-2ff3ad34ae95 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:52:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26573" uid: ea027591-5670-4a37-ac60-a9438b6fb608 kind: Event lastTimestamp: "2026-07-02T21:52:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:30Z" name: error-404-isvc-67ac0-predictor.18be978d3c073030 namespace: kserve-ci-e2e-test resourceVersion: "26927" uid: 94ea2633-9065-4b16-acaf-811eca4b9bcd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:52:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26573" uid: ea027591-5670-4a37-ac60-a9438b6fb608 kind: Event lastTimestamp: "2026-07-02T21:52:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:30Z" name: error-404-isvc-67ac0-predictor.18be978d3c6b95f9 namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 4e554e4d-5452-46cb-a64f-12b642fb7b4f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:52:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: ea027591-5670-4a37-ac60-a9438b6fb608 kind: Event lastTimestamp: "2026-07-02T21:53:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:53:00Z" name: error-404-isvc-67ac0-predictor.18be979439aed5d1 namespace: kserve-ci-e2e-test resourceVersion: "27194" uid: 620895f9-181a-438a-8b0c-fa7eba241d84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:52:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: ea027591-5670-4a37-ac60-a9438b6fb608 kind: Event lastTimestamp: "2026-07-02T21:53:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:53:00Z" name: error-404-isvc-67ac0-predictor.18be97943a197170 namespace: kserve-ci-e2e-test resourceVersion: "27197" uid: 7c9bdea8-1ac7-4144-a232-0a1d8f1b486c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 4b35ae3a-b3bc-4b1f-8bd0-bca52592c133 kind: Event lastTimestamp: "2026-07-02T21:52:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:48Z" name: error-404-isvc-67ac0.18be9789bda12894 namespace: kserve-ci-e2e-test resourceVersion: "27074" uid: b473a554-24ea-4aca-8bbd-9ac87e7ea3fa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 4b35ae3a-b3bc-4b1f-8bd0-bca52592c133 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-67ac0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-67ac0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0.18be9789bdebc210 namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: d3e3a9fe-ff2d-489d-80e4-07ba1184f172 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 4b35ae3a-b3bc-4b1f-8bd0-bca52592c133 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-67ac0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-67ac0.18be9789bdec0667 namespace: kserve-ci-e2e-test resourceVersion: "26579" uid: e3066308-30bb-41db-815b-321b4b6e0bae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "27071" uid: 4b35ae3a-b3bc-4b1f-8bd0-bca52592c133 kind: Event lastTimestamp: "2026-07-02T21:52:48Z" message: InferenceService [error-404-isvc-67ac0] is Ready metadata: creationTimestamp: "2026-07-02T21:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:48Z" name: error-404-isvc-67ac0.18be9794e930a403 namespace: kserve-ci-e2e-test resourceVersion: "27072" uid: a3dcbfd3-8b85-4fd7-b4ad-54bbc90be250 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:31:10.127196Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:31:10Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be9666a78dadb3 namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 5f3be5d2-5684-48fb-a9a2-081777092d89 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7615e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be9666b0fcb200 namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 01dddc55-cf0d-4f6d-b919-93eeb75e05f1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:31:11Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:31:11Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be9666e5075b14 namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 1b56f1db-ad50-4f94-947c-ca2e54317419 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:31:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:11Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be9666e687b21b namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 1170bffa-1201-4099-a4a9-225842162158 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:31:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:11Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be9666e9a2d85d namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: c6ced665-b030-4f1c-86da-c29b67a7566f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:31:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:11Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be9666ea180e5a namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: d1bc54c7-ced5-4a6b-a918-b1596083e8ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:31:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:11Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be9666ea227376 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: a131feb7-78de-47b8-9e5c-bb95e2f25e50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:31:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:11Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be9666ed528897 namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: a9a2cb71-8cb6-4b7a-8e52-72a7b2396eb6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:31:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:11Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be9666edd11964 namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: 0586772e-b468-4160-9d82-22f2ad914fe8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:31:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:48Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be966747fbb7bb namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: 5f235b2b-3ca6-4dda-ad9e-f60fe22df7a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:40:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:24Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be96e7d6c94355 namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: c4ec9117-a130-4187-b2fe-c27a55c7388b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 97b3a01d-fb66-4107-8fec-c45de02b4d92 kind: Event lastTimestamp: "2026-07-02T21:40:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:24Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj.18be96e7d6c9b4f6 namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: 807624d5-36bd-4cbe-80b9-747ff4edd3ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7615e-predictor-5bf56c7b5c namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: 146be07d-1e91-4252-808c-f32dbd176f0e kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: 'Created pod: error-404-isvc-7615e-predictor-5bf56c7b5c-fxmsj' metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:10Z" name: error-404-isvc-7615e-predictor-5bf56c7b5c.18be9666a7291698 namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 82e10d1a-257f-466e-b8fa-548c1f72a12f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: 3b181fdf-5d62-42c9-a878-82fc299723f7 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Scaled up replica set error-404-isvc-7615e-predictor-5bf56c7b5c from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:10Z" name: error-404-isvc-7615e-predictor.18be9666a55bd27e namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: 84332979-6c24-49bf-8ca4-1faa4d126fc7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:31:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 0ec477ae-4308-4a3d-8800-5edebffbf5a4 kind: Event lastTimestamp: "2026-07-02T21:31:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:55Z" name: error-404-isvc-7615e-predictor.18be966a23b18d9c namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: 9687fbf1-0c74-4477-861e-d4543887b5a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:31:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 0ec477ae-4308-4a3d-8800-5edebffbf5a4 kind: Event lastTimestamp: "2026-07-02T21:31:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:55Z" name: error-404-isvc-7615e-predictor.18be966a240da706 namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: b9f21cfd-2e62-45e5-a585-c3f3cc04be79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:32:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 0ec477ae-4308-4a3d-8800-5edebffbf5a4 kind: Event lastTimestamp: "2026-07-02T21:32:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:25Z" name: error-404-isvc-7615e-predictor.18be9674a01b1117 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: f65ec733-42bf-4870-a2d1-8c6483c9e077 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:32:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 0ec477ae-4308-4a3d-8800-5edebffbf5a4 kind: Event lastTimestamp: "2026-07-02T21:32:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:25Z" name: error-404-isvc-7615e-predictor.18be9674a08b3799 namespace: kserve-ci-e2e-test resourceVersion: "16756" uid: a3966acb-10ce-4bfe-ae1c-a7c3f988884e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7615e namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: bc42cbc3-2128-4936-a132-d1f8dfeecbf8 kind: Event lastTimestamp: "2026-07-02T21:31:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:58Z" name: error-404-isvc-7615e.18be9666a55e4e1f namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 460e137c-14a5-4aaf-aeb3-d8532256dcb2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7615e namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: bc42cbc3-2128-4936-a132-d1f8dfeecbf8 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-7615e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7615e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:10Z" name: error-404-isvc-7615e.18be9666a5d6d870 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: d9230169-fee7-4806-9070-f187ec37a899 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7615e namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: bc42cbc3-2128-4936-a132-d1f8dfeecbf8 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7615e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:10Z" name: error-404-isvc-7615e.18be9666a5d7112e namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: a69d371e-aa94-41f5-a2d9-cf0b68faa8ca reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7615e namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: bc42cbc3-2128-4936-a132-d1f8dfeecbf8 kind: Event lastTimestamp: "2026-07-02T21:31:58Z" message: InferenceService [error-404-isvc-7615e] is Ready metadata: creationTimestamp: "2026-07-02T21:31:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:58Z" name: error-404-isvc-7615e.18be967202f7d5e0 namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: cb18f7ee-10e2-472c-bfa2-6241e5f43dc4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:51:15.999204Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "25999" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-76a5f-predictor-578849c975-9gcw7 to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:51:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:51:16Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977f6b203292 namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 3da06164-b3f9-49ae-bb65-bab9353931a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T21:51:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-76a5f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:16Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977f721bdff2 namespace: kserve-ci-e2e-test resourceVersion: "26030" uid: 61ffa56c-55b2-4e51-a38e-5b429f355521 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26014" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:51:17Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977fa89eb0a2 namespace: kserve-ci-e2e-test resourceVersion: "26052" uid: 0c03139f-e94c-456a-b833-004356723df6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977faa69474f namespace: kserve-ci-e2e-test resourceVersion: "26055" uid: 25e13afb-abf6-48d4-ad44-b44bfe9e4340 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977fad9d3720 namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: 52c60654-9376-423d-bcf2-94e9af9dbe0d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977fae0d9b17 namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: a160c738-8eae-4201-be25-01aae0984076 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977fae18443d namespace: kserve-ci-e2e-test resourceVersion: "26060" uid: fe68713b-9719-4e22-8590-2a97e0419ee4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977fb0eaaa22 namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: dba003da-f304-47a6-9d5f-3a8fee128fb0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977fb15ad995 namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 400a6b7d-4cac-4e9a-add9-ea2fabbd5590 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:51:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T21:51:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:54Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be977fe3890a26 namespace: kserve-ci-e2e-test resourceVersion: "26435" uid: 6d40eb15-aaf9-46bb-9423-74840c7f5228 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T22:08:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T22:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:08:35Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be98716e10a8eb namespace: kserve-ci-e2e-test resourceVersion: "33715" uid: 81cd7634-eb45-46fb-a677-a9d5b19c0f86 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-76a5f-predictor-578849c975-9gcw7 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 80691226-a702-4432-b4c1-08223db2b3db kind: Event lastTimestamp: "2026-07-02T22:08:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T22:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:08:35Z" name: error-404-isvc-76a5f-predictor-578849c975-9gcw7.18be98716e117638 namespace: kserve-ci-e2e-test resourceVersion: "33716" uid: 345bdf30-4cb1-429b-b3f2-803efb54d09a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-76a5f-predictor-578849c975 namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: 99b5718b-b72b-4344-8426-8c7601337042 kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: 'Created pod: error-404-isvc-76a5f-predictor-578849c975-9gcw7' metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:15Z" name: error-404-isvc-76a5f-predictor-578849c975.18be977f6ab1a5df namespace: kserve-ci-e2e-test resourceVersion: "26001" uid: 723b6dd4-fa78-40f5-92da-3cb2ff3c087f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25989" uid: e22aa614-54de-4ce8-b443-f23df80f9ba4 kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: Scaled up replica set error-404-isvc-76a5f-predictor-578849c975 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:15Z" name: error-404-isvc-76a5f-predictor.18be977f68f64937 namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: cc4b1980-ab37-44ae-9909-a0d37eeda5a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 4547029f-99a5-4c59-9f94-be22b07e610f kind: Event lastTimestamp: "2026-07-02T21:51:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:45Z" name: error-404-isvc-76a5f-predictor.18be9782e768048e namespace: kserve-ci-e2e-test resourceVersion: "26336" uid: 5378c766-b271-4673-b8db-26b2dff6002e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 4547029f-99a5-4c59-9f94-be22b07e610f kind: Event lastTimestamp: "2026-07-02T21:51:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:45Z" name: error-404-isvc-76a5f-predictor.18be9782e7c956be namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: ce299e7d-a8a3-4d67-ac15-ad15b91d177c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26196" uid: 4547029f-99a5-4c59-9f94-be22b07e610f kind: Event lastTimestamp: "2026-07-02T21:52:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:31Z" name: error-404-isvc-76a5f-predictor.18be9789e507d872 namespace: kserve-ci-e2e-test resourceVersion: "26935" uid: 770bf4d4-9418-4a17-a6ed-b4960e71115e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:52:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26196" uid: 4547029f-99a5-4c59-9f94-be22b07e610f kind: Event lastTimestamp: "2026-07-02T21:52:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:52:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:31Z" name: error-404-isvc-76a5f-predictor.18be9789e56dbd65 namespace: kserve-ci-e2e-test resourceVersion: "26936" uid: d21c0d97-33ac-4a0d-8f1c-33c7712b21e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-76a5f namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 81a9d154-5573-48c4-9abe-1a2df4363e5e kind: Event lastTimestamp: "2026-07-02T21:52:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:04Z" name: error-404-isvc-76a5f.18be977f68fa6f59 namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: f273096c-8c78-41bb-b309-a7045719285e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-76a5f namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 81a9d154-5573-48c4-9abe-1a2df4363e5e kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-76a5f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-76a5f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:51:15Z" name: error-404-isvc-76a5f.18be977f6991e216 namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: ca16c621-40e1-4209-9cdf-c10db80fb877 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-76a5f namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 81a9d154-5573-48c4-9abe-1a2df4363e5e kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-76a5f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:51:15Z" name: error-404-isvc-76a5f.18be977f69923140 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 11b2c1bb-a43d-4ec2-b596-623c1f48d905 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-76a5f namespace: kserve-ci-e2e-test resourceVersion: "26669" uid: 81a9d154-5573-48c4-9abe-1a2df4363e5e kind: Event lastTimestamp: "2026-07-02T21:52:04Z" message: InferenceService [error-404-isvc-76a5f] is Ready metadata: creationTimestamp: "2026-07-02T21:52:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:04Z" name: error-404-isvc-76a5f.18be978a9e6a2ccd namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 239076b0-b7fd-4e28-bacb-7c9813fc0b47 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:41:35.225824Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:41:35Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f83255c23a namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: b8149ef4-786b-4ed1-97b0-290b14d7cf24 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:41:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : failed to sync secret cache: timed out waiting for the condition' metadata: creationTimestamp: "2026-07-02T21:41:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:36Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f87871d673 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: f8c9610e-fe52-4b5a-b4cc-f229830c4446 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:41:37Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:41:37Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f8a52b3e57 namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: d78a2b8a-42c4-4b77-98d4-6f3df62f1d8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:41:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:37Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f8a65677c9 namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: 04fd790b-6f6a-478b-9ec2-0f93ff1dc2a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:41:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:37Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f8a94c5651 namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 4f70915b-aade-44c0-81fd-f48a6bc26e6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:41:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:37Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f8a9cbd934 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 87bae408-eb2a-4e8c-a1b5-ee1b1b9ecac0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:41:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:37Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f8a9d719fe namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: 10600409-fd23-4ed3-82a4-f4a8a603a68b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:41:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:37Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f8ad84868f namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 1cff89d2-9f74-46bc-b7ee-cebfc05a9047 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:41:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:37Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f8adfad800 namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: 9a0c8ac4-d0b2-4282-9811-657f0df595a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:41:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:42:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:41:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:14Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be96f8f5d1e682 namespace: kserve-ci-e2e-test resourceVersion: "21629" uid: 95e9eb28-9fa9-452f-bd0b-ff329b75db34 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:50Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be977965690d9e namespace: kserve-ci-e2e-test resourceVersion: "25680" uid: ba9efaf5-1d8c-4a1e-9d01-2047a39bd3a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: cb66a45d-e080-4406-9218-0c4bf5107548 kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:50Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr.18be9779656962d2 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 0fc05705-23a3-4519-86e7-73908b07eb26 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-87dcc-predictor-5b4bfbcdc9 namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: f117580c-e9af-4897-94cb-35ded6fc87c3 kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: 'Created pod: error-404-isvc-87dcc-predictor-5b4bfbcdc9-c5zlr' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:35Z" name: error-404-isvc-87dcc-predictor-5b4bfbcdc9.18be96f83186277e namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 060d9ee5-c968-45a8-8fba-c768652768fb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: aec6b3f8-6120-412c-877f-1cb743f1f0f6 kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Scaled up replica set error-404-isvc-87dcc-predictor-5b4bfbcdc9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:35Z" name: error-404-isvc-87dcc-predictor.18be96f82f21cea0 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: e5f7a63a-152e-44f0-9112-ce932df0f6b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:41:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: 5fbb4b88-b002-47fe-a02b-b87f2062e487 kind: Event lastTimestamp: "2026-07-02T21:42:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:05Z" name: error-404-isvc-87dcc-predictor.18be96fbad58888f namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 4502e75d-cb3c-4b3f-aa98-f2dab9bce92f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:41:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: 5fbb4b88-b002-47fe-a02b-b87f2062e487 kind: Event lastTimestamp: "2026-07-02T21:42:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:05Z" name: error-404-isvc-87dcc-predictor.18be96fbadb989fa namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: 027e257e-c200-41c3-8513-2ce9503d456c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 5fbb4b88-b002-47fe-a02b-b87f2062e487 kind: Event lastTimestamp: "2026-07-02T21:42:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:35Z" name: error-404-isvc-87dcc-predictor.18be9702ab1bc50a namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: b9cf4a7b-2d73-40d7-9824-5022534f5ca6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 5fbb4b88-b002-47fe-a02b-b87f2062e487 kind: Event lastTimestamp: "2026-07-02T21:42:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:35Z" name: error-404-isvc-87dcc-predictor.18be9702ab84bddc namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 708f1f60-ab18-4461-9e28-59e3c76dc0d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87dcc namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 918cacf7-2f04-4c43-ac36-29ba6e0e662a kind: Event lastTimestamp: "2026-07-02T21:42:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:24Z" name: error-404-isvc-87dcc.18be96f82f22e17c namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: 6f7bd115-4acb-48c4-8857-821cb7c59541 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87dcc namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 918cacf7-2f04-4c43-ac36-29ba6e0e662a kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-87dcc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-87dcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:35Z" name: error-404-isvc-87dcc.18be96f82f954b0b namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: 86dbf50f-9d24-4c4d-80bd-ecdf3a64d916 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87dcc namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 918cacf7-2f04-4c43-ac36-29ba6e0e662a kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-87dcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:35Z" name: error-404-isvc-87dcc.18be96f82f959ecb namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: c144af5c-3ba6-46bd-837b-638516a5cc2c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87dcc namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 918cacf7-2f04-4c43-ac36-29ba6e0e662a kind: Event lastTimestamp: "2026-07-02T21:42:24Z" message: InferenceService [error-404-isvc-87dcc] is Ready metadata: creationTimestamp: "2026-07-02T21:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:24Z" name: error-404-isvc-87dcc.18be9703b1c4f607 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6b9105dd-c83f-47c4-992e-a13a1423a367 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:42:01.059976Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:42:01Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe362ba0bd namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: ffb6cb64-098e-44d3-b4b7-30293de8ea91 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8bf58-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe45f355c0 namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 450b57a7-690b-4a33-8591-178b152e85dc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:42:02Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:42:02Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe747eb8c5 namespace: kserve-ci-e2e-test resourceVersion: "21509" uid: 248af7e1-8f3b-494e-a3e9-19c45befaaff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:42:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:02Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe75fdb51c namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 09688528-3d9a-45ab-8521-1068272929f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:42:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:02Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe793fc67e namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: aa8ad8bd-9217-496c-877f-c5efa20cd903 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:42:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:02Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe79af4c9a namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: d88d481d-7983-4e52-b566-8fa2030dbd6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:42:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:02Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe79ba87ca namespace: kserve-ci-e2e-test resourceVersion: "21515" uid: f039ca0f-2d9f-4a5c-96f4-02991d8e335a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:42:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:02Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe7cea0a61 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: ac909769-e1ce-48e0-b163-7185aa8e2506 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:42:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:02Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe7d5af97b namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: d7e23acb-a109-44ed-b876-b9afeab2ae3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:42:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:51:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:18Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be96fe9307a4d6 namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: 84dfdbc1-3ba3-4cc9-ab53-52b55fe2485f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:51:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:16Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be977f6de9b4cc namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: e853160e-b4fa-48f9-a1dd-cfab8c68f41b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:51:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:16Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be977f6dec2707 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 14bb58a2-f9b8-4c90-b238-2400e6e3797b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 7b3bad8d-ffe0-4351-8e8e-0c1b13c833b7 kind: Event lastTimestamp: "2026-07-02T21:51:18Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:18Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm.18be9780076c9017 namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: ad442b72-0e6c-4a06-bdf6-75a14759c7e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8bf58-predictor-5d6f9ddb7f namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 1a016cf5-c439-49dd-a890-ffaea22453ee kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: 'Created pod: error-404-isvc-8bf58-predictor-5d6f9ddb7f-l7cxm' metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:01Z" name: error-404-isvc-8bf58-predictor-5d6f9ddb7f.18be96fe35b352b3 namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: bc21d4b2-fc88-442b-ad6c-458d4bf1e2fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 08dee225-a294-408a-b076-07ae1937cd93 kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: Scaled up replica set error-404-isvc-8bf58-predictor-5d6f9ddb7f from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:01Z" name: error-404-isvc-8bf58-predictor.18be96fe3415ed46 namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 88379619-8ec6-49a0-aa2a-a76b21f065b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 578167a5-f956-4d5b-9fcb-68cb751082c3 kind: Event lastTimestamp: "2026-07-02T21:42:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:31Z" name: error-404-isvc-8bf58-predictor.18be9701b28766f3 namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: 520558e2-aeac-475e-a179-79df18cd6cad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 578167a5-f956-4d5b-9fcb-68cb751082c3 kind: Event lastTimestamp: "2026-07-02T21:42:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:31Z" name: error-404-isvc-8bf58-predictor.18be9701b2ee8b34 namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: 5e7109cf-a996-4cdf-8d70-b1a50bbe29b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:42:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 578167a5-f956-4d5b-9fcb-68cb751082c3 kind: Event lastTimestamp: "2026-07-02T21:43:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:16Z" name: error-404-isvc-8bf58-predictor.18be9708b04713c3 namespace: kserve-ci-e2e-test resourceVersion: "22218" uid: 0fc27dfc-8bd0-4d7f-a342-6740b3a08b91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:42:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 578167a5-f956-4d5b-9fcb-68cb751082c3 kind: Event lastTimestamp: "2026-07-02T21:43:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:16Z" name: error-404-isvc-8bf58-predictor.18be9708b0af9580 namespace: kserve-ci-e2e-test resourceVersion: "22219" uid: 4c38802f-d95b-4268-930e-5eb95445d882 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 1baddb6e-9d60-42b7-9e44-5dc7997d1e75 kind: Event lastTimestamp: "2026-07-02T21:42:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:48Z" name: error-404-isvc-8bf58.18be96fe342eecd3 namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: af741645-0f2f-43c9-a5a5-bd5c109fad93 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 1baddb6e-9d60-42b7-9e44-5dc7997d1e75 kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-8bf58": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8bf58": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:01Z" name: error-404-isvc-8bf58.18be96fe34a058ac namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: 6a1b67d6-273b-4ced-b1b2-213b5540f54f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 1baddb6e-9d60-42b7-9e44-5dc7997d1e75 kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8bf58": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:01Z" name: error-404-isvc-8bf58.18be96fe34a0b1b1 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 600923a1-2716-4dd6-ac9a-db1ad9a70640 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 1baddb6e-9d60-42b7-9e44-5dc7997d1e75 kind: Event lastTimestamp: "2026-07-02T21:42:48Z" message: InferenceService [error-404-isvc-8bf58] is Ready metadata: creationTimestamp: "2026-07-02T21:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:48Z" name: error-404-isvc-8bf58.18be97094dc9598c namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: 9f199dc3-91c0-4dff-8e23-8b0f9c5bb721 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:31:35.928975Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:31:35Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966ca97587a4 namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: b39216c0-1b40-40fb-bd1e-0c71461764ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a3b0c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966cba14d007 namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 7fbff856-262e-45cb-b343-9c1fcfc2b868 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:31:36Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966ce7469a6b namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 0207c8df-d435-4b47-81ef-41a25ba7f7ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966ce8c6c03e namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: be3127bd-392f-40e4-8083-e9d3b284f753 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:31:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:37Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966ced07fac9 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 164a2b22-d4e0-4df7-8b4f-c1aa38f3f3cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:31:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:37Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966ced7eaa38 namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 7921902d-d7bd-4ead-8778-7747bdac034f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:31:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:37Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966ced897094 namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: 23ae26dc-0178-4663-8962-9daa3fcd892f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:31:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:37Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966cf0bf04d0 namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 8e4e63b0-3002-44af-9ebe-1c349d1f4dc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:31:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:37Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966cf136cf38 namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: d6a2fd97-2683-41a7-bed4-98e55180cce8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:31:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:40:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:54Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be966d5cbd6066 namespace: kserve-ci-e2e-test resourceVersion: "20588" uid: 0cbe8862-74db-4b93-8e5b-1fed490d0007 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:50Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be96ede27141e7 namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: 0b20b7f1-b971-4abe-ae00-c75e938c9b11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:50Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be96ede2727717 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 0f40b8de-a58d-4eec-9671-bf07be3b0229 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 4518a8cb-7f8f-4421-b588-92c52963712b kind: Event lastTimestamp: "2026-07-02T21:40:54Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:54Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm.18be96eed122ce16 namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: b4476aff-a848-4601-912b-efaadd28e057 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a3b0c-predictor-5dd5b5fb79 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 1c51fd37-09e5-493b-837b-3c73956e7707 kind: Event lastTimestamp: "2026-07-02T21:31:35Z" message: 'Created pod: error-404-isvc-a3b0c-predictor-5dd5b5fb79-sl8qm' metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:35Z" name: error-404-isvc-a3b0c-predictor-5dd5b5fb79.18be966ca90b9524 namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 591ee615-535b-4484-a276-07aee8754ee8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: c905bab4-cebc-4a9d-8513-cf65500268d4 kind: Event lastTimestamp: "2026-07-02T21:31:35Z" message: Scaled up replica set error-404-isvc-a3b0c-predictor-5dd5b5fb79 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:35Z" name: error-404-isvc-a3b0c-predictor.18be966ca73f305a namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 0d7b8b58-69a6-42d6-9c93-76fbfabe79ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: 4b6da4c6-9213-47d8-92d5-c49e5f2ba044 kind: Event lastTimestamp: "2026-07-02T21:31:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:50Z" name: error-404-isvc-a3b0c-predictor.18be967025af94ae namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: d0ad3c79-efa0-4917-b3ff-22a201f4f098 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: 4b6da4c6-9213-47d8-92d5-c49e5f2ba044 kind: Event lastTimestamp: "2026-07-02T21:31:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:50Z" name: error-404-isvc-a3b0c-predictor.18be9670260fb6cc namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: cdb4f2be-5216-4daa-9298-27b648edf29c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:32:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 4b6da4c6-9213-47d8-92d5-c49e5f2ba044 kind: Event lastTimestamp: "2026-07-02T21:32:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:32:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:50Z" name: error-404-isvc-a3b0c-predictor.18be9673a4a51261 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: 1cf1973d-708f-4e39-afd9-e41aa4ea306d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:32:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 4b6da4c6-9213-47d8-92d5-c49e5f2ba044 kind: Event lastTimestamp: "2026-07-02T21:32:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:32:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:50Z" name: error-404-isvc-a3b0c-predictor.18be9673a5126d72 namespace: kserve-ci-e2e-test resourceVersion: "16991" uid: f7e088ad-ea10-44be-b250-6cf578c5c151 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 28976a1f-5597-4730-b775-93f4406e7ce2 kind: Event lastTimestamp: "2026-07-02T21:32:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:25Z" name: error-404-isvc-a3b0c.18be966ca7408eec namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: 49413ddf-a3eb-49d4-bc1f-4609e07c3510 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16747" uid: 28976a1f-5597-4730-b775-93f4406e7ce2 kind: Event lastTimestamp: "2026-07-02T21:32:25Z" message: InferenceService [error-404-isvc-a3b0c] is Ready metadata: creationTimestamp: "2026-07-02T21:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:25Z" name: error-404-isvc-a3b0c.18be9678178f92cc namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 7ebe33ef-cda9-4f5b-ad13-77130a7e4582 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:50:50.089106Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:50:50Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be977962c382c3 namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: 60c6f329-ff9e-4b5d-a925-d82f9a34c309 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c9d16-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:50Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be97796ebd6b6f namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 7a3313ed-da1b-4e38-9d9c-f232854f9338 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:50:51Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9779a04d962e namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: 2360364e-7dcc-44c2-ae63-a57b3a7cec8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9779a1a05583 namespace: kserve-ci-e2e-test resourceVersion: "25714" uid: e0c3b2f6-4eaf-4f45-b950-4dbc25ea2463 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9779a48ace3a namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: a1d3ff86-d3fd-4859-bfc1-e5567f425f21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9779a502056a namespace: kserve-ci-e2e-test resourceVersion: "25719" uid: a67e7bc3-fb0f-4317-94ac-3057cea8c151 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9779a50bfdbd namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: 318a07db-e49a-4d10-a2fa-b2a0d69662c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9779aa76d7cf namespace: kserve-ci-e2e-test resourceVersion: "25721" uid: 89763326-77a6-45ce-a192-d0513799ff7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9779aaedfa16 namespace: kserve-ci-e2e-test resourceVersion: "25722" uid: 72146272-0e16-48d5-a86b-6d237540ed5c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:51:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:27Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9779ce51299f namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: 0fb8745d-8ea7-48a9-a933-6de946876bee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9789b6045e4d namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: e6bfa00e-c5ff-4451-b2a1-fe008c65d04d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be9789b608537f namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: b90528d8-f429-42e5-abfe-76f18228c261 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 741d6758-2db9-420b-aaae-08ad6ff4c4e6 kind: Event lastTimestamp: "2026-07-02T21:52:02Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:52:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:02Z" name: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl.18be978a567675c2 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: e9570043-da8c-467e-821e-2daba5061b15 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c9d16-predictor-d5f45cf87 namespace: kserve-ci-e2e-test resourceVersion: "25650" uid: 0dcc98f0-c821-4026-b8ea-5d9faa0df33d kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: 'Created pod: error-404-isvc-c9d16-predictor-d5f45cf87-t2ssl' metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:50:50Z" name: error-404-isvc-c9d16-predictor-d5f45cf87.18be97796247159e namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 1361fa89-a871-458d-84a4-7be8379444e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: a3d97ac4-02b9-4d7a-828a-b0c08407ffa1 kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: Scaled up replica set error-404-isvc-c9d16-predictor-d5f45cf87 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:50:50Z" name: error-404-isvc-c9d16-predictor.18be977960a4c075 namespace: kserve-ci-e2e-test resourceVersion: "25651" uid: aa894bb0-7308-4d98-829a-387c5dd169df reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: 16656f42-47f8-49ec-b114-a96721cf1844 kind: Event lastTimestamp: "2026-07-02T21:51:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:20Z" name: error-404-isvc-c9d16-predictor.18be977cdf17d49c namespace: kserve-ci-e2e-test resourceVersion: "26108" uid: 37fc6c94-d651-4b93-a947-375a93809a07 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: 16656f42-47f8-49ec-b114-a96721cf1844 kind: Event lastTimestamp: "2026-07-02T21:51:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:20Z" name: error-404-isvc-c9d16-predictor.18be977cdf97b3ff namespace: kserve-ci-e2e-test resourceVersion: "26109" uid: ed38fd64-af1e-4ff8-8112-896eb5eefe56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25841" uid: 16656f42-47f8-49ec-b114-a96721cf1844 kind: Event lastTimestamp: "2026-07-02T21:51:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:50Z" name: error-404-isvc-c9d16-predictor.18be9783dce7417a namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: 9ff940cc-07cd-453a-a7df-989a270d4003 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25841" uid: 16656f42-47f8-49ec-b114-a96721cf1844 kind: Event lastTimestamp: "2026-07-02T21:51:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:50Z" name: error-404-isvc-c9d16-predictor.18be9783dd4d5a43 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 2910617f-e8b7-4240-9de6-cb6de06a622a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "25616" uid: 73fd6c81-01eb-4524-98e0-564d5f2780a3 kind: Event lastTimestamp: "2026-07-02T21:51:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:51:38Z" name: error-404-isvc-c9d16.18be977960af63d9 namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 4bb57728-1ea2-45d2-aa19-b064db4fd5f4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "25616" uid: 73fd6c81-01eb-4524-98e0-564d5f2780a3 kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-c9d16": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c9d16": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:50:50Z" name: error-404-isvc-c9d16.18be9779614f63e6 namespace: kserve-ci-e2e-test resourceVersion: "25655" uid: 0e3d7adf-7011-4a62-95d4-9509889d279f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "25616" uid: 73fd6c81-01eb-4524-98e0-564d5f2780a3 kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c9d16": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:50:50Z" name: error-404-isvc-c9d16.18be9779614f9af0 namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: f1c534f4-3ec6-4921-a52f-b151400cea27 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "26262" uid: 73fd6c81-01eb-4524-98e0-564d5f2780a3 kind: Event lastTimestamp: "2026-07-02T21:51:37Z" message: InferenceService [error-404-isvc-c9d16] is Ready metadata: creationTimestamp: "2026-07-02T21:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:51:37Z" name: error-404-isvc-c9d16.18be9784895b732e namespace: kserve-ci-e2e-test resourceVersion: "26263" uid: 075d1dd5-2b1d-4ac1-809b-61f2218bb4a1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:29:25.903851Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e36cd-predictor-6986f87bbc-4v972 to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:29:25Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be964e635b9cd4 namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: 42a5ae29-aff9-4b59-a45e-c42ce9bfd904 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e36cd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be964e6e70ec63 namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: ab25bdad-33ce-4dfa-bf14-324243635e8b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:29:26Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be964ea0e24982 namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 92c69f5b-7941-4632-ba98-6ef211ed5f12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be964ea26c2e7b namespace: kserve-ci-e2e-test resourceVersion: "14731" uid: 819e16e3-3f1c-49d8-834c-a425aa514567 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:38Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" in 11.634s (11.634s including waiting). Image size: 1334700636 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:38Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be965157eb69d6 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 96a233f6-8abc-4d96-b850-086346dfef4e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:29:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:38Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be96515b9121ac namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 38f5e832-f98a-40ac-91d6-810a622f944e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:29:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:38Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be96515c09589f namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 154cd2cf-f204-47e2-ab5a-dbc9af234ecc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T21:29:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:38Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be96515c171bd6 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 6db39d78-cf65-4f52-89d7-0912e5c1caab reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:40Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.208s (2.208s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:40Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be9651dfb6f1b8 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 75ca52c0-65ce-47a2-91b2-4925dbb2fa4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:40Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be9651e3125313 namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 04c602d6-25bc-437b-870b-76d84a4bbb9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:29:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:40Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be9651e38e4d2a namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: f968b05d-b579-4083-b930-b273945c9461 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:30:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:18Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be96523c5e3e12 namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 5086596c-e2ca-4fbf-8fec-a4e5844171fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be9666ab0d679b namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: b76b07ce-a188-4ed2-95c6-fb2242914c0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be9666ab0dd55b namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: b04d57fc-42c6-463d-9247-b19e2c29043a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 00b187fc-8887-420e-a767-e1142ff1f0f2 kind: Event lastTimestamp: "2026-07-02T21:31:14Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-02T21:31:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:14Z" name: error-404-isvc-e36cd-predictor-6986f87bbc-4v972.18be9667a82b4012 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: 70568356-259a-446b-9cb5-569412ebd689 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e36cd-predictor-6986f87bbc namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: 68add514-c1a5-4944-b879-8677c45575ee kind: Event lastTimestamp: "2026-07-02T21:29:25Z" message: 'Created pod: error-404-isvc-e36cd-predictor-6986f87bbc-4v972' metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:25Z" name: error-404-isvc-e36cd-predictor-6986f87bbc.18be964e62f9e627 namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: 7feb83ab-1943-44ba-9bda-5e69cb6a6551 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: 84504594-50de-40ba-b9f0-0261b1b7ab51 kind: Event lastTimestamp: "2026-07-02T21:29:25Z" message: Scaled up replica set error-404-isvc-e36cd-predictor-6986f87bbc from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:25Z" name: error-404-isvc-e36cd-predictor.18be964e60062937 namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 9ebc6d66-f3bc-45f5-a185-e48a687faebe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 80b7b074-58d3-4efe-9724-f79af0520517 kind: Event lastTimestamp: "2026-07-02T21:30:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:10Z" name: error-404-isvc-e36cd-predictor.18be9651dea50b0d namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 1ee82ae7-9a0d-4d11-9ede-7473de686d86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 80b7b074-58d3-4efe-9724-f79af0520517 kind: Event lastTimestamp: "2026-07-02T21:30:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:10Z" name: error-404-isvc-e36cd-predictor.18be9651df08db38 namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: e210980d-11c5-468f-99a9-69a2def04170 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 80b7b074-58d3-4efe-9724-f79af0520517 kind: Event lastTimestamp: "2026-07-02T21:30:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:55Z" name: error-404-isvc-e36cd-predictor.18be965c5adbad12 namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: 5fd51de2-a658-4bd8-a215-b524df64b5a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 80b7b074-58d3-4efe-9724-f79af0520517 kind: Event lastTimestamp: "2026-07-02T21:30:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:55Z" name: error-404-isvc-e36cd-predictor.18be965c5b3af95e namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: 6a5caed7-5435-4ac8-bb6c-96264f83946f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e36cd namespace: kserve-ci-e2e-test resourceVersion: "14596" uid: 57370a3a-56f4-4e1e-8ecf-6360bd0a4a2b kind: Event lastTimestamp: "2026-07-02T21:30:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:28Z" name: error-404-isvc-e36cd.18be964e603773ec namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: 7398869a-4a6c-4923-a01a-d91d8f291592 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e36cd namespace: kserve-ci-e2e-test resourceVersion: "15315" uid: 57370a3a-56f4-4e1e-8ecf-6360bd0a4a2b kind: Event lastTimestamp: "2026-07-02T21:30:28Z" message: InferenceService [error-404-isvc-e36cd] is Ready metadata: creationTimestamp: "2026-07-02T21:30:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:28Z" name: error-404-isvc-e36cd.18be965cf75af646 namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: 9ce9fbdf-f171-4033-9459-25deeecaf29f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:40:24.922962Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20143" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ee70d-predictor-5797bfd779-l6tvl to ip-10-0-130-121.ec2.internal metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:40:24Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e7d3f5402f namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: 32adeeba-e065-40d3-bfbf-be8fe1cb2c6a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ee70d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:25Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e7e0a60e3e namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: 4dc491d3-9709-4987-80eb-d123b7a72b11 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:40:25Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e81169ff03 namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: 2e7ca915-5963-4c3e-8301-afc113f48b97 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:25Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e812bc77aa namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 07a2c5d9-6a5e-4407-af81-69ee3488725b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:40:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:26Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e815b0aa20 namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: 2682bfba-5059-4f4e-9bae-3c0a07ead10f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:40:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:26Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e8162013a0 namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 3c166f56-b078-4d54-83d9-de4ee2b8cbb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:40:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:26Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e81629ae66 namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: 7801e3c4-0660-4753-82da-ed7c5a0125d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:40:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:26Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e81997d265 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: 0d187411-565e-4142-83b1-a2c7cf3bcb6f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:40:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:26Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e81a25df3b namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 6636c010-d60b-48a1-8a29-74adbb8a0011 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:41:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:02Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96e81e4d9a60 namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: d59fdd27-e218-4fd6-ac9b-9fbb2532b4fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96f825e23d3a namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: 1a5c65b8-f194-42b4-8853-683c8fcc91be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96f825e2fe82 namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: 22095d65-005e-42c4-a397-0860d9924337 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: c9624fa5-352c-4268-b4d6-a1668d4aa00f kind: Event lastTimestamp: "2026-07-02T21:41:37Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:37Z" name: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl.18be96f8a6660b23 namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: 2d6b07ab-63b8-449a-a664-faf4dab2a696 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-121.ec2.internal source: component: kubelet host: ip-10-0-130-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ee70d-predictor-5797bfd779 namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: cf0ee701-9deb-4057-91a6-f22565b22581 kind: Event lastTimestamp: "2026-07-02T21:40:24Z" message: 'Created pod: error-404-isvc-ee70d-predictor-5797bfd779-l6tvl' metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:24Z" name: error-404-isvc-ee70d-predictor-5797bfd779.18be96e7d389bac4 namespace: kserve-ci-e2e-test resourceVersion: "20144" uid: fa44c50f-3e2c-4c58-9fd7-c9fa631e5c1a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: e98002e3-555e-4764-8cdf-d33eaf60ca33 kind: Event lastTimestamp: "2026-07-02T21:40:24Z" message: Scaled up replica set error-404-isvc-ee70d-predictor-5797bfd779 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:24Z" name: error-404-isvc-ee70d-predictor.18be96e7d1d54543 namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: 84875734-5774-4bc4-a532-4131155e4c9d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: 7962371a-48f7-48b7-be42-6c1654ab9285 kind: Event lastTimestamp: "2026-07-02T21:40:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:39Z" name: error-404-isvc-ee70d-predictor.18be96eb5049cb5b namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 9e575cf4-400d-4e50-9263-6046b821a9d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: 7962371a-48f7-48b7-be42-6c1654ab9285 kind: Event lastTimestamp: "2026-07-02T21:40:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:39Z" name: error-404-isvc-ee70d-predictor.18be96eb50ac40c7 namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: b14cb88f-f008-4232-9084-e45f5ee4cbec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 7962371a-48f7-48b7-be42-6c1654ab9285 kind: Event lastTimestamp: "2026-07-02T21:41:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:24Z" name: error-404-isvc-ee70d-predictor.18be96eecf507bfa namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 1b722d7f-c814-4536-aaff-3ca174679b8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 7962371a-48f7-48b7-be42-6c1654ab9285 kind: Event lastTimestamp: "2026-07-02T21:41:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:24Z" name: error-404-isvc-ee70d-predictor.18be96eecfb57f31 namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: 8b814e07-0f69-4973-b82f-469eda2e4e19 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ee70d namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: f1dc89a2-7867-4fd7-926f-fb641b9d4cd1 kind: Event lastTimestamp: "2026-07-02T21:41:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:12Z" name: error-404-isvc-ee70d.18be96e7d1dd7c12 namespace: kserve-ci-e2e-test resourceVersion: "20751" uid: ce80f70a-fa88-4c98-870d-b89139a46a05 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ee70d namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: f1dc89a2-7867-4fd7-926f-fb641b9d4cd1 kind: Event lastTimestamp: "2026-07-02T21:41:12Z" message: InferenceService [error-404-isvc-ee70d] is Ready metadata: creationTimestamp: "2026-07-02T21:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:12Z" name: error-404-isvc-ee70d.18be96f2d947dd99 namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: bbd10645-d0f3-4c21-97c4-911cc53ffd02 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:29:25.811597Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:29:25Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be964e5ddc42e0 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: 4736c6a4-131c-4259-b638-e25a20b8c325 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be964e770512bd namespace: kserve-ci-e2e-test resourceVersion: "14691" uid: 80c0f482-6714-41ba-b4e4-9c24cc0725fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be964e795538b1 namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: 522f451f-344c-4a86-843c-04a7c8407b9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be964e7ccb1d91 namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: 70f7c04a-4d69-4d50-8386-2e9fb7aaf60e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be964e7d6f1cec namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: ee89eb96-fdd4-4131-a5ee-97ceef12b468 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:31Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" metadata: creationTimestamp: "2026-07-02T21:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:31Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be964fc0e77d8c namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: b7ca2080-c181-4449-ab6c-14cc14c01442 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" in 10.041s (10.041s including waiting). Image size: 1555748775 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be9652176ba757 namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 2bb5dc5b-24c5-4d6c-a133-2fdf70a5e6a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be96521bffcdf6 namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: 930d259a-3f40-44cf-910b-04f3fa55a9d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be96521c903f83 namespace: kserve-ci-e2e-test resourceVersion: "14896" uid: bde835d5-974f-4d2a-8792-6fb064966067 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be96521c9cedb1 namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 27739048-bf3d-4e41-a863-8e6aa8a06a7b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:46Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.736s (4.736s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:46Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be965336ebb599 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: 79a90cfd-15fb-41b4-9268-3e0c6c3ad034 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:47Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be96534d5b0a92 namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: a6f645e2-661b-496c-99b3-f5a2c709a59f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:29:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:47Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be96534e8d8f8c namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: 17cd4819-2f62-42a7-b549-5129bf392e63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:30:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:53Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be965382f55fb1 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 3bd193c5-b4af-4493-ab55-d4cb40811639 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:31:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:35Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be966ca2fb3f81 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 7e9b919f-c9e6-448f-8daa-9d6178a2fbcb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:31:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:35Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be966ca2fba0a3 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: fe90d468-8850-4fce-9ee1-a27944e75719 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: d30a040f-b5b2-4ef4-9cc9-a389f7c7c83b kind: Event lastTimestamp: "2026-07-02T21:31:38Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:38Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4.18be966d5b59bc0d namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 75ad4456-bf46-474e-a1be-d4ab890da857 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-766fdfb6bb namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 9cf30e0a-c347-49fb-bde6-cf7db7a2016a kind: Event lastTimestamp: "2026-07-02T21:29:25Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-766fdfb6bb-9hls4' metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:25Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb.18be964e5d6b3825 namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: c79dea37-79c7-4bb7-836c-a58763251809 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: c127971e-3059-4f55-9e04-29f4be124422 kind: Event lastTimestamp: "2026-07-02T21:29:25Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-766fdfb6bb from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:25Z" name: isvc-sklearn-graph-1-predictor.18be964e5a3b9ed3 namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: d666d485-5e6d-449a-ac5e-c82775b4c709 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: 117faccc-38e8-4082-adac-e62bf7f0c1c1 kind: Event lastTimestamp: "2026-07-02T21:30:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:25Z" name: isvc-sklearn-graph-1-predictor.18be9651d89757e4 namespace: kserve-ci-e2e-test resourceVersion: "15278" uid: 84af3d01-e20b-4330-9a1d-d14adc6be8be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: 117faccc-38e8-4082-adac-e62bf7f0c1c1 kind: Event lastTimestamp: "2026-07-02T21:30:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:25Z" name: isvc-sklearn-graph-1-predictor.18be9651d9066a32 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: ed0929ee-4690-4f51-a569-8e1498d162be reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:30:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: 117faccc-38e8-4082-adac-e62bf7f0c1c1 kind: Event lastTimestamp: "2026-07-02T21:31:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:25Z" name: isvc-sklearn-graph-1-predictor.18be9658d64dbb0b namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: 7b11db2e-ce99-45ce-80c9-0a2a6186d2b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:30:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: 117faccc-38e8-4082-adac-e62bf7f0c1c1 kind: Event lastTimestamp: "2026-07-02T21:31:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:25Z" name: isvc-sklearn-graph-1-predictor.18be9658d6b82ca3 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: 0872f803-b841-43e7-9372-f4f1825fc140 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 5ceae141-f6c9-4a93-a641-0fc5e2727b64 kind: Event lastTimestamp: "2026-07-02T21:31:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:04Z" name: isvc-sklearn-graph-1.18be964e5a6551a8 namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: f76719eb-1680-4412-b2d9-a6acc20f3cf2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: 5ceae141-f6c9-4a93-a641-0fc5e2727b64 kind: Event lastTimestamp: "2026-07-02T21:31:04Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-02T21:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:04Z" name: isvc-sklearn-graph-1.18be96653aa8208c namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: e2a101ab-d840-4a63-b308-12792a8d9e6d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:29:26.112334Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be964e6fc8ca78 namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: da4680e2-2044-4757-8115-2e5387f08c23 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be964e8b368029 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: f1d82956-0b9e-4c3e-99f9-aad07ed06f86 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be964e8ce30fcd namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: 849ba77a-b9c7-4e62-b26e-cc32a68dc30c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be964e91d7719e namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: a00a3ac9-5c01-4595-97d7-41ed188f19ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be964e93c8f365 namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: 0dd57c26-8a28-45f8-b475-f29b71222e5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:31Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" metadata: creationTimestamp: "2026-07-02T21:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:31Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be964fc0d0eeaf namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: 21e8dd9a-6a02-4859-90a9-ec5d0af10ac8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" in 10.03s (10.03s including waiting). Image size: 1555748775 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be965216b18e95 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 71d0a1bf-54fb-4446-9c3f-312e4861ed45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be96521bab6daf namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: a3fd45e5-7b8c-4e03-81c2-341d649ea7e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be96521c6c67e6 namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: c9bafb2b-419a-443a-bd7d-51e5b3ef181d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be96521c7a3de8 namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 229db03e-2403-435f-a229-328cb298cec9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:46Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.992s (4.992s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:46Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be96534613bbb3 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: 188ec089-0cc6-49ff-b523-5b529199636d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:47Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be96534d98cd79 namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: dab4b9f7-b010-4c9b-beb5-1491c5010298 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:29:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:47Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be96534ed112e1 namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 759993f4-777c-428a-819a-24bb89d47df9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:29:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:31:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:40Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be9653beea4a03 namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 4a9bdfa3-558a-42b0-bf9d-c2368798e37c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be966cb06a2b22 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 26ed2838-d3aa-4ce2-a3cf-334366e2fe44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be966cb06adeaa namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 9adeeffa-e311-472e-a2ac-d47732b0e857 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 07c2fae2-0515-4e71-82a5-e8da54ad9ea7 kind: Event lastTimestamp: "2026-07-02T21:31:39Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:39Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2.18be966d974f986c namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: 69f4014a-5c70-4256-bafc-09eadf164c46 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-767d6d7f5d namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 4f0046fe-d297-4921-b7e2-233ed5fd0f11 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-767d6d7f5d-6p6t2' metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d.18be964e6f4e9fd5 namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: 44f24f43-967a-4817-971d-3c65b2a31e93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 03028491-94a8-49aa-a545-a5f2bd6b8fa8 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-767d6d7f5d from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:26Z" name: isvc-sklearn-graph-2-predictor.18be964e6c6a6d18 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 9209ad2b-b29b-480e-aaaa-9d3d9a0a39e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: e3cb12eb-dc44-4230-879e-ac3409356496 kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-2-predictor.18be9651ead8745e namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: 9bef20b6-e56b-47ff-91ec-f4e397d7eef0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: e3cb12eb-dc44-4230-879e-ac3409356496 kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:41Z" name: isvc-sklearn-graph-2-predictor.18be9651eb3ea15b namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a92b63e8-c047-4933-9d36-090a85f55455 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:29:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: e3cb12eb-dc44-4230-879e-ac3409356496 kind: Event lastTimestamp: "2026-07-02T21:31:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:26Z" name: isvc-sklearn-graph-2-predictor.18be965569ebba1c namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 6ff7ef64-cd8e-40f4-bfaa-cd55d3231718 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:29:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: e3cb12eb-dc44-4230-879e-ac3409356496 kind: Event lastTimestamp: "2026-07-02T21:31:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:26Z" name: isvc-sklearn-graph-2-predictor.18be96556a4ffcfc namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5edb7e94-1905-497b-9a3a-669f8f1d2a3a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 0aea884d-1623-4d19-9318-10be1cb4c16a kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:10Z" name: isvc-sklearn-graph-2.18be964e6c6a4da5 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 73d5abd2-9203-4d7e-bf6b-09c664feb788 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 0aea884d-1623-4d19-9318-10be1cb4c16a kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:10Z" name: isvc-sklearn-graph-2.18be9666b07dcc46 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 80e7b981-fcf4-43f9-975a-e6d38f417510 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:29:26.010600Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-445c7 to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:29:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be964e69b8862e namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 6e2bcd60-8acc-4114-a34f-4728bdf67403 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be964e7886a1a2 namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: 3d0b3298-5895-4d05-9cda-d3f36969a1cc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:29:27Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:29:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be964ead1a148d namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: cd2062c0-3d50-4d54-96f0-228a425257d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:29:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be964eaefe49bb namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 56b6dd9d-8371-42c1-859d-2a7dd8acb2d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:29:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be964eb3b415c4 namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 1dab7e0f-c345-4023-bcc4-9cf267b88a5c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:29:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be964eb482da65 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: b0c14985-59df-4653-8db9-e98c7efbd6f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:29:31Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T21:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be964fc0bc8fa6 namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 524c985b-ddb5-4cb3-ac28-3b5241029c09 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:30:01Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 29.517s (29.517s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T21:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be9656a01e70a0 namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: 1b2b9d30-7a11-4108-a5b3-b2a6cd7bdbca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:30:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be9656a33fc397 namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: e19803bc-3a1c-428c-b385-5a387b890b1b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:30:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be9656a3b48e63 namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: 187e59e1-f403-4ce8-8ec0-ab35e97ae16e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:30:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be9656a3bddb19 namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 26bf890b-7df1-4d20-b5f3-55efa2fcb466 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:30:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be9656a6ce7795 namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: 4f1406ae-f8d2-4131-900f-8a565d86f98e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:30:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be9656a75f4492 namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: bb4cefa2-fd81-4075-8c04-f8ee32889c2c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:31:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be965705dd6c98 namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: f2d1a68b-80e6-4417-8e36-0daa83022838 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:31:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be966ca9f59772 namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 637be0e8-558d-4b47-b7b0-81b84db41fce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:31:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be966ca9f5e350 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: d0e78a2d-cc6f-4719-83fe-5d3b156c57cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-445c7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: d294529b-b6a5-4b4a-ac86-8db2ab88adae kind: Event lastTimestamp: "2026-07-02T21:31:38Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-445c7.18be966d6006d342 namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: eb5d3e80-7a97-433f-a7bf-52427e50f15c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: 05c49a27-cf2b-42e9-b0e7-f97854da5659 kind: Event lastTimestamp: "2026-07-02T21:29:26Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-445c7' metadata: creationTimestamp: "2026-07-02T21:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:26Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be964e694b189e namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 334c23b4-d151-49e2-9b49-744bc37715c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 12238087-b390-46c0-9ba5-9c6f12fe2bcd kind: Event lastTimestamp: "2026-07-02T21:29:25Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:25Z" name: isvc-xgboost-graph-predictor.18be964e6647c812 namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: 92a6193a-fd6d-4d73-8361-e0d53e0e19b7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: c7660520-b92f-4b36-9fd2-ac0300371703 kind: Event lastTimestamp: "2026-07-02T21:30:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:41Z" name: isvc-xgboost-graph-predictor.18be9651e4e7de73 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 53797444-c8fd-440b-af5f-28efe7f92872 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: c7660520-b92f-4b36-9fd2-ac0300371703 kind: Event lastTimestamp: "2026-07-02T21:30:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:41Z" name: isvc-xgboost-graph-predictor.18be9651e548fd45 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: f2ea9a10-f739-4022-83eb-7bf7c54eb679 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:30:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: c7660520-b92f-4b36-9fd2-ac0300371703 kind: Event lastTimestamp: "2026-07-02T21:31:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:26Z" name: isvc-xgboost-graph-predictor.18be96635e997ec9 namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: c85422d8-c0f5-47bb-88a6-58673ad75449 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:30:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: c7660520-b92f-4b36-9fd2-ac0300371703 kind: Event lastTimestamp: "2026-07-02T21:31:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:26Z" name: isvc-xgboost-graph-predictor.18be96635efd2bfc namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 9c63c2b0-c809-43db-b2ce-03afccc540f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 4b7221a5-2972-4347-a635-7b02d33872dd kind: Event lastTimestamp: "2026-07-02T21:31:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:09Z" name: isvc-xgboost-graph.18be964e6677d4b8 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 8e113367-a843-4301-b72d-d93e788f02e0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: 4b7221a5-2972-4347-a635-7b02d33872dd kind: Event lastTimestamp: "2026-07-02T21:31:09Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-02T21:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:09Z" name: isvc-xgboost-graph.18be966669164638 namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: 8f3a00df-d88c-433d-9439-99d5a280f404 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:31:25.564892Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77cb85d899-bjbw7 namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: 9f667116-a685-46d7-9c79-96e9a15fc44c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-77cb85d899-bjbw7 to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:31:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:31:25Z" name: model-chainer-77cb85d899-bjbw7.18be966a3fb633d4 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: 2646e5d7-157c-4bc0-afef-2fcd29e9a476 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:31:25Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77cb85d899-bjbw7 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 9f667116-a685-46d7-9c79-96e9a15fc44c kind: Event lastTimestamp: "2026-07-02T21:31:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:26Z" name: model-chainer-77cb85d899-bjbw7.18be966a4d595b07 namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: 924a11a2-24b0-4a02-b01f-5e42854e38b9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77cb85d899-bjbw7 namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: 9f667116-a685-46d7-9c79-96e9a15fc44c kind: Event lastTimestamp: "2026-07-02T21:31:27Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:31:27Z" name: model-chainer-77cb85d899-bjbw7.18be966ab2186021 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 84d8201f-daf9-4f66-9722-7c7cd6fef9c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77cb85d899-bjbw7 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 9f667116-a685-46d7-9c79-96e9a15fc44c kind: Event lastTimestamp: "2026-07-02T21:31:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:27Z" name: model-chainer-77cb85d899-bjbw7.18be966ab3427919 namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 91f1c9da-868a-4f58-adfb-dca57cd6d095 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77cb85d899-bjbw7 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 9f667116-a685-46d7-9c79-96e9a15fc44c kind: Event lastTimestamp: "2026-07-02T21:31:27Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-02T21:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:27Z" name: model-chainer-77cb85d899-bjbw7.18be966ab5d2351b namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: ccef1cf1-3965-4171-b9af-dffd3f4e5e56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77cb85d899-bjbw7 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 9f667116-a685-46d7-9c79-96e9a15fc44c kind: Event lastTimestamp: "2026-07-02T21:31:27Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-02T21:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:27Z" name: model-chainer-77cb85d899-bjbw7.18be966ab698ad01 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: d425f46e-b5d5-4781-a5dd-397cf95e7135 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77cb85d899-bjbw7 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 9f667116-a685-46d7-9c79-96e9a15fc44c kind: Event lastTimestamp: "2026-07-02T21:31:35Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:35Z" name: model-chainer-77cb85d899-bjbw7.18be966c9a3a8dc1 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: c3e3f7a6-ae72-4085-8a6d-e0597b92b350 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:31:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77cb85d899-bjbw7 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 9f667116-a685-46d7-9c79-96e9a15fc44c kind: Event lastTimestamp: "2026-07-02T21:32:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:04Z" name: model-chainer-77cb85d899-bjbw7.18be966d8015564d namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: 319dc676-3f41-49c0-8568-0943887460cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-77cb85d899 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: 4a916493-6d68-4a0c-868c-e323115ace3b kind: Event lastTimestamp: "2026-07-02T21:31:25Z" message: 'Created pod: model-chainer-77cb85d899-bjbw7' metadata: creationTimestamp: "2026-07-02T21:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:25Z" name: model-chainer-77cb85d899.18be966a3f4eaf98 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 11b405e7-c6b8-4581-9196-a419261fe44e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 10ac8100-6df4-4d8a-9486-4b7aa23eae7e kind: Event lastTimestamp: "2026-07-02T21:31:25Z" message: Scaled up replica set model-chainer-77cb85d899 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:25Z" name: model-chainer.18be966a3e91cb39 namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 6eb9ffd2-33aa-49c0-9abd-469c8adafe2b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 07b920e2-e530-4b55-80cb-b2d65496dd47 kind: Event lastTimestamp: "2026-07-02T21:31:25Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-02T21:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:25Z" name: model-chainer.18be966a43c5202d namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: fe93653c-64ba-4555-91f5-f02689409002 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:41:50.731218Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1702f-5b5fd878cd-fpbw2 namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: 35dcffdc-a454-4dfb-8900-70ce88aee475 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-1702f-5b5fd878cd-fpbw2 to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:41:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:41:50Z" name: sequence-graph-1702f-5b5fd878cd-fpbw2.18be96fbce8741ed namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: a9194b6d-9889-4698-8839-7685567891c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1702f-5b5fd878cd-fpbw2 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 35dcffdc-a454-4dfb-8900-70ce88aee475 kind: Event lastTimestamp: "2026-07-02T21:41:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-1702f-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:50Z" name: sequence-graph-1702f-5b5fd878cd-fpbw2.18be96fbd8d7fb7d namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 9ebf3a71-dbe1-46b4-ae59-808cbf87ba6d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:51Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1702f-5b5fd878cd-fpbw2 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 35dcffdc-a454-4dfb-8900-70ce88aee475 kind: Event lastTimestamp: "2026-07-02T21:41:51Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:41:51Z" name: sequence-graph-1702f-5b5fd878cd-fpbw2.18be96fc0bc72f68 namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: f2bbcd9d-fb84-404b-bf7a-986284f17f65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1702f} kind: Pod name: sequence-graph-1702f-5b5fd878cd-fpbw2 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 35dcffdc-a454-4dfb-8900-70ce88aee475 kind: Event lastTimestamp: "2026-07-02T21:41:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:51Z" name: sequence-graph-1702f-5b5fd878cd-fpbw2.18be96fc0d0f81c3 namespace: kserve-ci-e2e-test resourceVersion: "21294" uid: efc0ca05-7d7e-40a4-9d74-4b848bfb3db8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1702f} kind: Pod name: sequence-graph-1702f-5b5fd878cd-fpbw2 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 35dcffdc-a454-4dfb-8900-70ce88aee475 kind: Event lastTimestamp: "2026-07-02T21:41:51Z" message: 'Created container: sequence-graph-1702f' metadata: creationTimestamp: "2026-07-02T21:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:51Z" name: sequence-graph-1702f-5b5fd878cd-fpbw2.18be96fc10896a9e namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: 0bce8735-8b57-42ac-8364-1d4bf83a3f83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1702f} kind: Pod name: sequence-graph-1702f-5b5fd878cd-fpbw2 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 35dcffdc-a454-4dfb-8900-70ce88aee475 kind: Event lastTimestamp: "2026-07-02T21:41:51Z" message: Started container sequence-graph-1702f metadata: creationTimestamp: "2026-07-02T21:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:51Z" name: sequence-graph-1702f-5b5fd878cd-fpbw2.18be96fc10f618aa namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 03ea2c92-32f3-4c42-a1d0-4f38cbafd951 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1702f} kind: Pod name: sequence-graph-1702f-5b5fd878cd-fpbw2 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 35dcffdc-a454-4dfb-8900-70ce88aee475 kind: Event lastTimestamp: "2026-07-02T21:42:00Z" message: Stopping container sequence-graph-1702f metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:00Z" name: sequence-graph-1702f-5b5fd878cd-fpbw2.18be96fe2829cd9e namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 66a29dfe-b06d-4e85-b84e-d3743df36a4f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1702f} kind: Pod name: sequence-graph-1702f-5b5fd878cd-fpbw2 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 35dcffdc-a454-4dfb-8900-70ce88aee475 kind: Event lastTimestamp: "2026-07-02T21:42:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:28Z" name: sequence-graph-1702f-5b5fd878cd-fpbw2.18be96fedbbaf006 namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 0c2ddbc8-f168-45b5-9897-61178595996d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-1702f-5b5fd878cd namespace: kserve-ci-e2e-test resourceVersion: "21261" uid: 498f7968-703f-4617-8c63-2da6caef0021 kind: Event lastTimestamp: "2026-07-02T21:41:50Z" message: 'Created pod: sequence-graph-1702f-5b5fd878cd-fpbw2' metadata: creationTimestamp: "2026-07-02T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:50Z" name: sequence-graph-1702f-5b5fd878cd.18be96fbce160687 namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: 16a8cdbc-9ee8-477a-9f60-c108341486d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-1702f namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: 83e21339-982e-4088-9155-b688d2d2a883 kind: Event lastTimestamp: "2026-07-02T21:41:50Z" message: Scaled up replica set sequence-graph-1702f-5b5fd878cd from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:50Z" name: sequence-graph-1702f.18be96fbcd705fb5 namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: 4d12e464-3e82-464f-a154-b1c5968d89b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-1702f namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: 17bdc577-21ee-444e-ba9b-f703d672586c kind: Event lastTimestamp: "2026-07-02T21:41:50Z" message: InferenceGraph [sequence-graph-1702f] is Ready metadata: creationTimestamp: "2026-07-02T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:50Z" name: sequence-graph-1702f.18be96fbd207775d namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: 84e75b3a-b5c6-4f7d-ac58-7a7271636629 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:43:01.030613Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8bf58-998d46b48-pwhn2 namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: 2daee948-6dcb-4e95-8598-a66d01bbaea4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8bf58-998d46b48-pwhn2 to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58-998d46b48-pwhn2.18be970c2cb2ecbd namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: d4031250-b202-4af0-aa13-47686243e648 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:01Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8bf58-998d46b48-pwhn2 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: 2daee948-6dcb-4e95-8598-a66d01bbaea4 kind: Event lastTimestamp: "2026-07-02T21:43:01Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58-998d46b48-pwhn2.18be970c46308eb6 namespace: kserve-ci-e2e-test resourceVersion: "22104" uid: 4f900b82-826d-4ec2-94dd-bf8023383092 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bf58} kind: Pod name: sequence-graph-8bf58-998d46b48-pwhn2 namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: 2daee948-6dcb-4e95-8598-a66d01bbaea4 kind: Event lastTimestamp: "2026-07-02T21:43:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58-998d46b48-pwhn2.18be970c4769fcd6 namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: d7d150f2-c33f-41a6-8f8e-8ab312bbb642 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bf58} kind: Pod name: sequence-graph-8bf58-998d46b48-pwhn2 namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: 2daee948-6dcb-4e95-8598-a66d01bbaea4 kind: Event lastTimestamp: "2026-07-02T21:43:01Z" message: 'Created container: sequence-graph-8bf58' metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58-998d46b48-pwhn2.18be970c4ad26242 namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: d497e12b-0ace-40d4-ac7e-347bc06fec0d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bf58} kind: Pod name: sequence-graph-8bf58-998d46b48-pwhn2 namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: 2daee948-6dcb-4e95-8598-a66d01bbaea4 kind: Event lastTimestamp: "2026-07-02T21:43:01Z" message: Started container sequence-graph-8bf58 metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58-998d46b48-pwhn2.18be970c4b5ff7a1 namespace: kserve-ci-e2e-test resourceVersion: "22109" uid: 8ea90282-a9d2-4e64-a2d0-ec2169bde1ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bf58} kind: Pod name: sequence-graph-8bf58-998d46b48-pwhn2 namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: 2daee948-6dcb-4e95-8598-a66d01bbaea4 kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: Stopping container sequence-graph-8bf58 metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:15Z" name: sequence-graph-8bf58-998d46b48-pwhn2.18be977f5b9e7360 namespace: kserve-ci-e2e-test resourceVersion: "25934" uid: 5f5e3b1f-fcd9-4a80-8831-af62f0cbd89a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:51:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bf58} kind: Pod name: sequence-graph-8bf58-998d46b48-pwhn2 namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: 2daee948-6dcb-4e95-8598-a66d01bbaea4 kind: Event lastTimestamp: "2026-07-02T21:51:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:43Z" name: sequence-graph-8bf58-998d46b48-pwhn2.18be977fe80b6f20 namespace: kserve-ci-e2e-test resourceVersion: "26311" uid: bb0858c4-072a-435e-8b8f-4585b1429c85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8bf58-998d46b48 namespace: kserve-ci-e2e-test resourceVersion: "22075" uid: ec3b1255-5029-4bf7-87d1-fdcb01eba5ab kind: Event lastTimestamp: "2026-07-02T21:43:01Z" message: 'Created pod: sequence-graph-8bf58-998d46b48-pwhn2' metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58-998d46b48.18be970c2c3eb7ff namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: b9304240-1221-4aae-9766-d4758fa0fede reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "22072" uid: 9cc72885-c1ae-464a-80be-5ba4987fa21f kind: Event lastTimestamp: "2026-07-02T21:43:01Z" message: Scaled up replica set sequence-graph-8bf58-998d46b48 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58.18be970c2b85d45c namespace: kserve-ci-e2e-test resourceVersion: "22076" uid: 4ba02f73-bccd-4e9e-b3a4-23820f08b316 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "22093" uid: bc7e8f38-e7e0-4262-a4d3-799811e000ae kind: Event lastTimestamp: "2026-07-02T21:43:01Z" message: InferenceGraph [sequence-graph-8bf58] is Ready metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58.18be970c3094dc14 namespace: kserve-ci-e2e-test resourceVersion: "22095" uid: 167135df-9c57-472e-a1d2-c2da201ad3ce reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: bc7e8f38-e7e0-4262-a4d3-799811e000ae kind: Event lastTimestamp: "2026-07-02T21:43:01Z" message: 'Failed to update status for InferenceGraph "sequence-graph-8bf58": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-8bf58": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58.18be970c35d61333 namespace: kserve-ci-e2e-test resourceVersion: "22098" uid: d3059405-4163-4b3c-8100-23fc797a0f49 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: bc7e8f38-e7e0-4262-a4d3-799811e000ae kind: Event lastTimestamp: "2026-07-02T21:43:01Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-8bf58": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:43:01Z" name: sequence-graph-8bf58.18be970c35d65cff namespace: kserve-ci-e2e-test resourceVersion: "22099" uid: 547ed798-c2bd-4110-9f25-910fd50aa946 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: 33aea82b-bad9-48c2-b08b-a6cf534ad660 kind: Event lastTimestamp: "2026-07-02T21:43:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:16Z" name: sequence-graph-8bf58.18be970fa9f5200c namespace: kserve-ci-e2e-test resourceVersion: "22215" uid: 72ff502c-5997-481d-af28-8826bc909130 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: 33aea82b-bad9-48c2-b08b-a6cf534ad660 kind: Event lastTimestamp: "2026-07-02T21:43:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:16Z" name: sequence-graph-8bf58.18be970faa4d2256 namespace: kserve-ci-e2e-test resourceVersion: "22217" uid: 070646cd-a290-4f28-83c4-d13a040981a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "22216" uid: 33aea82b-bad9-48c2-b08b-a6cf534ad660 kind: Event lastTimestamp: "2026-07-02T21:43:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:31Z" name: sequence-graph-8bf58.18be971328ea9611 namespace: kserve-ci-e2e-test resourceVersion: "22324" uid: 8e6dc627-8f74-420c-a6a1-c43cc0a19d43 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "22216" uid: 33aea82b-bad9-48c2-b08b-a6cf534ad660 kind: Event lastTimestamp: "2026-07-02T21:43:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:31Z" name: sequence-graph-8bf58.18be9713292f364d namespace: kserve-ci-e2e-test resourceVersion: "22326" uid: ca9c400f-c548-4ce3-85f2-94aed861c87f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:32:35.856342Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3b0c-c794dd67c-4k8h6 namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: 28179935-2e9d-4d95-b5bd-78edd9d21658 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a3b0c-c794dd67c-4k8h6 to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:32:35Z" name: sequence-graph-a3b0c-c794dd67c-4k8h6.18be967a9d68982f namespace: kserve-ci-e2e-test resourceVersion: "16848" uid: 089ec4c1-5e60-4366-a12c-c8d650ef6a43 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3b0c-c794dd67c-4k8h6 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 28179935-2e9d-4d95-b5bd-78edd9d21658 kind: Event lastTimestamp: "2026-07-02T21:32:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-a3b0c-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:36Z" name: sequence-graph-a3b0c-c794dd67c-4k8h6.18be967aa7d07d04 namespace: kserve-ci-e2e-test resourceVersion: "16864" uid: b22ea615-55f0-4e2d-b189-1b44e03c3092 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3b0c-c794dd67c-4k8h6 namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: 28179935-2e9d-4d95-b5bd-78edd9d21658 kind: Event lastTimestamp: "2026-07-02T21:32:36Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:32:36Z" name: sequence-graph-a3b0c-c794dd67c-4k8h6.18be967ada661ce6 namespace: kserve-ci-e2e-test resourceVersion: "16873" uid: 6f5810f5-8cba-41f0-a9f7-1523ce0af733 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3b0c} kind: Pod name: sequence-graph-a3b0c-c794dd67c-4k8h6 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 28179935-2e9d-4d95-b5bd-78edd9d21658 kind: Event lastTimestamp: "2026-07-02T21:32:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:36Z" name: sequence-graph-a3b0c-c794dd67c-4k8h6.18be967adb957c1b namespace: kserve-ci-e2e-test resourceVersion: "16875" uid: 0c4d2935-8c95-458a-9d7d-0ef715bc25c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3b0c} kind: Pod name: sequence-graph-a3b0c-c794dd67c-4k8h6 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 28179935-2e9d-4d95-b5bd-78edd9d21658 kind: Event lastTimestamp: "2026-07-02T21:32:36Z" message: 'Created container: sequence-graph-a3b0c' metadata: creationTimestamp: "2026-07-02T21:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:36Z" name: sequence-graph-a3b0c-c794dd67c-4k8h6.18be967adedaddb7 namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: f818815b-626f-4ab6-89f2-2a716a63d8cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3b0c} kind: Pod name: sequence-graph-a3b0c-c794dd67c-4k8h6 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 28179935-2e9d-4d95-b5bd-78edd9d21658 kind: Event lastTimestamp: "2026-07-02T21:32:36Z" message: Started container sequence-graph-a3b0c metadata: creationTimestamp: "2026-07-02T21:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:36Z" name: sequence-graph-a3b0c-c794dd67c-4k8h6.18be967adf4a18a4 namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: 44ac963b-d6d6-4127-8d31-868410d7cac5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3b0c} kind: Pod name: sequence-graph-a3b0c-c794dd67c-4k8h6 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 28179935-2e9d-4d95-b5bd-78edd9d21658 kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: Stopping container sequence-graph-a3b0c metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:50Z" name: sequence-graph-a3b0c-c794dd67c-4k8h6.18be96edce488019 namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: dd3a8500-4817-4868-a35b-3cef0d6c2529 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3b0c} kind: Pod name: sequence-graph-a3b0c-c794dd67c-4k8h6 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 28179935-2e9d-4d95-b5bd-78edd9d21658 kind: Event lastTimestamp: "2026-07-02T21:41:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:18Z" name: sequence-graph-a3b0c-c794dd67c-4k8h6.18be96ee8cc61a10 namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: 1477782b-3760-4b53-a7eb-af2077f13468 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a3b0c-c794dd67c namespace: kserve-ci-e2e-test resourceVersion: "16840" uid: d7ab26b6-ece0-4d7e-8e40-15dfa9d08f8d kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: 'Created pod: sequence-graph-a3b0c-c794dd67c-4k8h6' metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:35Z" name: sequence-graph-a3b0c-c794dd67c.18be967a9d002a6f namespace: kserve-ci-e2e-test resourceVersion: "16845" uid: 33da3868-d9de-4c5c-bf93-4fd42429e27a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: b3321cfe-f82b-4795-86cc-d340cd03c62a kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: Scaled up replica set sequence-graph-a3b0c-c794dd67c from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:35Z" name: sequence-graph-a3b0c.18be967a9c5c4afd namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: 8a5b12c8-f797-4ea5-9a94-b9effbe356a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: 359c602c-7825-454c-80ce-c3064537932f kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: InferenceGraph [sequence-graph-a3b0c] is Ready metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:35Z" name: sequence-graph-a3b0c.18be967aa140aa96 namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: df5582be-0f5a-4592-a44c-e1feedc6e2c4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 5a7c9e30-4eb0-41e4-8410-be3f621579d8 kind: Event lastTimestamp: "2026-07-02T21:32:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:50Z" name: sequence-graph-a3b0c.18be967e1b187363 namespace: kserve-ci-e2e-test resourceVersion: "16985" uid: b7d60479-4276-43a3-a2d3-8e044ff8e5de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 5a7c9e30-4eb0-41e4-8410-be3f621579d8 kind: Event lastTimestamp: "2026-07-02T21:32:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:50Z" name: sequence-graph-a3b0c.18be967e1b78f178 namespace: kserve-ci-e2e-test resourceVersion: "16987" uid: fe624bcc-f45b-4e53-94ee-e22a4bc4d175 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:33:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16986" uid: 5a7c9e30-4eb0-41e4-8410-be3f621579d8 kind: Event lastTimestamp: "2026-07-02T21:33:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:05Z" name: sequence-graph-a3b0c.18be96819a1cd301 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 627df50c-a0ee-484b-8fff-68a526977aa2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:33:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16986" uid: 5a7c9e30-4eb0-41e4-8410-be3f621579d8 kind: Event lastTimestamp: "2026-07-02T21:33:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:05Z" name: sequence-graph-a3b0c.18be96819a689869 namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: 27f8a0b0-a4f4-438a-8f77-75d37b4b4ce4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:53:00.332515Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-67ac0-5f65757465-2w9sc namespace: kserve-ci-e2e-test resourceVersion: "27179" uid: ff7485bd-3a1b-4d01-a7fb-4645671780f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-67ac0-5f65757465-2w9sc to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:53:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:53:00Z" name: splitter-graph-67ac0-5f65757465-2w9sc.18be9797b5e040bd namespace: kserve-ci-e2e-test resourceVersion: "27184" uid: 1c68ad1c-573b-4dfc-bfe6-9eed6a891c4a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:53:00Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-67ac0-5f65757465-2w9sc namespace: kserve-ci-e2e-test resourceVersion: "27189" uid: ff7485bd-3a1b-4d01-a7fb-4645671780f1 kind: Event lastTimestamp: "2026-07-02T21:53:00Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:53:00Z" name: splitter-graph-67ac0-5f65757465-2w9sc.18be9797cf28f81a namespace: kserve-ci-e2e-test resourceVersion: "27202" uid: 93569ee6-625e-4050-9972-95217af5b16e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-67ac0} kind: Pod name: splitter-graph-67ac0-5f65757465-2w9sc namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: ff7485bd-3a1b-4d01-a7fb-4645671780f1 kind: Event lastTimestamp: "2026-07-02T21:53:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:53:00Z" name: splitter-graph-67ac0-5f65757465-2w9sc.18be9797d067d5f0 namespace: kserve-ci-e2e-test resourceVersion: "27205" uid: 27c4fd9c-5e44-4043-b6a1-a1716451df52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-67ac0} kind: Pod name: splitter-graph-67ac0-5f65757465-2w9sc namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: ff7485bd-3a1b-4d01-a7fb-4645671780f1 kind: Event lastTimestamp: "2026-07-02T21:53:00Z" message: 'Created container: splitter-graph-67ac0' metadata: creationTimestamp: "2026-07-02T21:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:53:00Z" name: splitter-graph-67ac0-5f65757465-2w9sc.18be9797d3566b19 namespace: kserve-ci-e2e-test resourceVersion: "27206" uid: 96e8c138-7bbd-49db-9b13-380f0fc4f57b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-67ac0} kind: Pod name: splitter-graph-67ac0-5f65757465-2w9sc namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: ff7485bd-3a1b-4d01-a7fb-4645671780f1 kind: Event lastTimestamp: "2026-07-02T21:53:00Z" message: Started container splitter-graph-67ac0 metadata: creationTimestamp: "2026-07-02T21:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:53:00Z" name: splitter-graph-67ac0-5f65757465-2w9sc.18be9797d3c7ae99 namespace: kserve-ci-e2e-test resourceVersion: "27207" uid: 8bd06e35-041a-4ffe-98d0-a2203fd43a7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:01:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-67ac0} kind: Pod name: splitter-graph-67ac0-5f65757465-2w9sc namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: ff7485bd-3a1b-4d01-a7fb-4645671780f1 kind: Event lastTimestamp: "2026-07-02T22:01:14Z" message: Stopping container splitter-graph-67ac0 metadata: creationTimestamp: "2026-07-02T22:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:01:14Z" name: splitter-graph-67ac0-5f65757465-2w9sc.18be980adef1643d namespace: kserve-ci-e2e-test resourceVersion: "30636" uid: 016965eb-d8b0-4fdd-a35b-d4e9ea22a610 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T22:01:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-67ac0} kind: Pod name: splitter-graph-67ac0-5f65757465-2w9sc namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: ff7485bd-3a1b-4d01-a7fb-4645671780f1 kind: Event lastTimestamp: "2026-07-02T22:01:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T22:01:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:01:42Z" name: splitter-graph-67ac0-5f65757465-2w9sc.18be980b6b903d96 namespace: kserve-ci-e2e-test resourceVersion: "30871" uid: 655d5fca-7334-4fc5-a039-05cb7a746356 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:53:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-67ac0-5f65757465 namespace: kserve-ci-e2e-test resourceVersion: "27175" uid: b89efe71-566a-471c-aaff-c5a9cc2eac09 kind: Event lastTimestamp: "2026-07-02T21:53:00Z" message: 'Created pod: splitter-graph-67ac0-5f65757465-2w9sc' metadata: creationTimestamp: "2026-07-02T21:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:53:00Z" name: splitter-graph-67ac0-5f65757465.18be9797b5699073 namespace: kserve-ci-e2e-test resourceVersion: "27181" uid: 84690f64-b661-4fd1-a747-43d83e18d43c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:53:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "27172" uid: cfc09d86-2c5a-4b32-8fff-b0fedf29269e kind: Event lastTimestamp: "2026-07-02T21:53:00Z" message: Scaled up replica set splitter-graph-67ac0-5f65757465 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:53:00Z" name: splitter-graph-67ac0.18be9797b49723cd namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: bda4577a-74fe-427c-9c18-6a1d36637d68 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:53:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "27195" uid: 67b03357-b665-4ec2-a80e-6132e7b27602 kind: Event lastTimestamp: "2026-07-02T21:53:00Z" message: InferenceGraph [splitter-graph-67ac0] is Ready metadata: creationTimestamp: "2026-07-02T21:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:53:00Z" name: splitter-graph-67ac0.18be9797b95fa438 namespace: kserve-ci-e2e-test resourceVersion: "27198" uid: 59312d46-e207-45f0-942f-c433f0f9ea69 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:53:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "27174" uid: 3f25e922-d474-4a03-8eba-acf9908644c0 kind: Event lastTimestamp: "2026-07-02T21:53:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:53:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:53:30Z" name: splitter-graph-67ac0.18be979b32f32a80 namespace: kserve-ci-e2e-test resourceVersion: "27417" uid: 6e8cb939-2b95-49cf-90d9-a58f15304343 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:53:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "27174" uid: 3f25e922-d474-4a03-8eba-acf9908644c0 kind: Event lastTimestamp: "2026-07-02T21:53:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:53:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:53:30Z" name: splitter-graph-67ac0.18be979b33341c9d namespace: kserve-ci-e2e-test resourceVersion: "27418" uid: 2db48529-8c29-4864-90b7-7b05029bce8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:51:50.038410Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c9d16-7bdf897478-wv2sr namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 4c08cdc4-cf12-4810-80cf-10e1cf34efbd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-c9d16-7bdf897478-wv2sr to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:51:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:51:50Z" name: splitter-graph-c9d16-7bdf897478-wv2sr.18be978758054e22 namespace: kserve-ci-e2e-test resourceVersion: "26382" uid: dda9aafe-5f88-4c64-a524-7b7ae76f8464 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:50Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c9d16-7bdf897478-wv2sr namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 4c08cdc4-cf12-4810-80cf-10e1cf34efbd kind: Event lastTimestamp: "2026-07-02T21:51:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-c9d16-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:51:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:50Z" name: splitter-graph-c9d16-7bdf897478-wv2sr.18be978764b5609a namespace: kserve-ci-e2e-test resourceVersion: "26400" uid: 651adfa2-8b5f-4827-abe3-d3ba27f67fdd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c9d16-7bdf897478-wv2sr namespace: kserve-ci-e2e-test resourceVersion: "26385" uid: 4c08cdc4-cf12-4810-80cf-10e1cf34efbd kind: Event lastTimestamp: "2026-07-02T21:51:51Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:51:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:51:51Z" name: splitter-graph-c9d16-7bdf897478-wv2sr.18be9787951adf8a namespace: kserve-ci-e2e-test resourceVersion: "26406" uid: 696dd9cf-4b04-4cdf-b2ba-713b43b5384c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c9d16} kind: Pod name: splitter-graph-c9d16-7bdf897478-wv2sr namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 4c08cdc4-cf12-4810-80cf-10e1cf34efbd kind: Event lastTimestamp: "2026-07-02T21:51:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:51:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:51Z" name: splitter-graph-c9d16-7bdf897478-wv2sr.18be9787965fd9fc namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: f391b495-2b30-4660-a5c6-7bcd572361f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c9d16} kind: Pod name: splitter-graph-c9d16-7bdf897478-wv2sr namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 4c08cdc4-cf12-4810-80cf-10e1cf34efbd kind: Event lastTimestamp: "2026-07-02T21:51:51Z" message: 'Created container: splitter-graph-c9d16' metadata: creationTimestamp: "2026-07-02T21:51:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:51Z" name: splitter-graph-c9d16-7bdf897478-wv2sr.18be9787991dd971 namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: 361057f1-071c-40f4-95f1-5af2096cedcd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c9d16} kind: Pod name: splitter-graph-c9d16-7bdf897478-wv2sr namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 4c08cdc4-cf12-4810-80cf-10e1cf34efbd kind: Event lastTimestamp: "2026-07-02T21:51:51Z" message: Started container splitter-graph-c9d16 metadata: creationTimestamp: "2026-07-02T21:51:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:51Z" name: splitter-graph-c9d16-7bdf897478-wv2sr.18be9787999099e3 namespace: kserve-ci-e2e-test resourceVersion: "26412" uid: 701e4b2b-4baf-49c9-a147-9e35550f9b7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c9d16} kind: Pod name: splitter-graph-c9d16-7bdf897478-wv2sr namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 4c08cdc4-cf12-4810-80cf-10e1cf34efbd kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Stopping container splitter-graph-c9d16 metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: splitter-graph-c9d16-7bdf897478-wv2sr.18be9789b0a3c160 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: d09ee09f-b468-4414-93aa-5cfdf3ac135a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:52:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c9d16} kind: Pod name: splitter-graph-c9d16-7bdf897478-wv2sr namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 4c08cdc4-cf12-4810-80cf-10e1cf34efbd kind: Event lastTimestamp: "2026-07-02T21:52:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:52:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:28Z" name: splitter-graph-c9d16-7bdf897478-wv2sr.18be978a5efdd03b namespace: kserve-ci-e2e-test resourceVersion: "26905" uid: ade05716-c567-4199-b25a-07b2db45a5af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-c9d16-7bdf897478 namespace: kserve-ci-e2e-test resourceVersion: "26372" uid: 1d6ecd89-8c5f-4570-81bf-adafb18086f5 kind: Event lastTimestamp: "2026-07-02T21:51:50Z" message: 'Created pod: splitter-graph-c9d16-7bdf897478-wv2sr' metadata: creationTimestamp: "2026-07-02T21:51:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:50Z" name: splitter-graph-c9d16-7bdf897478.18be9787577669ba namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: e4943d24-bcdc-45f5-9f40-cc5910e53ac9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "26369" uid: 68d5c135-a5de-4512-82c0-b12b0ae5a070 kind: Event lastTimestamp: "2026-07-02T21:51:50Z" message: Scaled up replica set splitter-graph-c9d16-7bdf897478 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:51:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:50Z" name: splitter-graph-c9d16.18be978756ad0bf5 namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 730c5f8a-bc76-41a3-8103-5417656de0bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "26389" uid: 0b5dac56-e2d6-4b1b-aa9d-36d9af3c0f5e kind: Event lastTimestamp: "2026-07-02T21:51:50Z" message: InferenceGraph [splitter-graph-c9d16] is Ready metadata: creationTimestamp: "2026-07-02T21:51:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:51:50Z" name: splitter-graph-c9d16.18be97875ba3b53d namespace: kserve-ci-e2e-test resourceVersion: "26391" uid: cf99f746-8bd3-4d77-91e2-9545e424b36a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:40:50.752524Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1702f-predictor-d85f8dbd4-57r4s to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:40:50Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96edd7850156 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: f9c67d52-ee8d-4f6e-a36d-e3ee58c05fae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1702f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:50Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96ede34b4db7 namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 975c98f2-dd97-41ce-89a4-35c7d22d17e6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:40:51Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96ee1497dc3c namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 99a5c589-4596-474c-89d5-1e1ebfc7e367 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96ee15dbc400 namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: 74449d7f-b2de-4432-9d03-c5780eba88ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96ee197b071f namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: ec7436ab-8f53-4e34-8e4e-4962d7f030a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96ee19ea5eb6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: c5bc2b19-3881-4b0e-948b-57cb7fadef15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96ee19f31210 namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: df2d00e8-5d6f-4fc7-8f65-9256d85c5702 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96ee1d2530ae namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 0b45b08f-0146-4adc-9c91-9c961251776a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96ee1d95262f namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: 56944139-176d-4085-bad6-dd7ad8d5f1ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:41:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:29Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96ee889bf9cd namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: 9de17beb-64d7-4a6b-864c-9e36aa755aef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:42:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:00Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96fe2f792683 namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: 15a9abe0-d196-49b8-94b5-581a7c2306d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: 08b59517-c227-4b47-9824-7ce0c25e6707 kind: Event lastTimestamp: "2026-07-02T21:42:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:00Z" name: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s.18be96fe2f7999cd namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: fe9f6212-9454-4980-b844-b1febc517179 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1702f-predictor-d85f8dbd4 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 5126ecdd-5ea8-4a17-96f2-3e20dd419db1 kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: 'Created pod: success-200-isvc-1702f-predictor-d85f8dbd4-57r4s' metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:50Z" name: success-200-isvc-1702f-predictor-d85f8dbd4.18be96edd67aac15 namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: ce8802b2-a603-4cf1-8598-d85415a6af61 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: 848d8113-ba00-4103-a6df-449a48a76002 kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: Scaled up replica set success-200-isvc-1702f-predictor-d85f8dbd4 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:50Z" name: success-200-isvc-1702f-predictor.18be96edd39d5620 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: d777d889-d69c-4cab-bd6c-b3a8d41ac553 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: b172e143-ef2d-4de5-b632-e79816240026 kind: Event lastTimestamp: "2026-07-02T21:41:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:05Z" name: success-200-isvc-1702f-predictor.18be96f1520c7df8 namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: d0c4cc9f-3701-42da-9175-a09835f8a5e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: b172e143-ef2d-4de5-b632-e79816240026 kind: Event lastTimestamp: "2026-07-02T21:41:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:05Z" name: success-200-isvc-1702f-predictor.18be96f1526ee373 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 392076e6-54bf-4616-959b-b220dfecb59a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:41:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: b172e143-ef2d-4de5-b632-e79816240026 kind: Event lastTimestamp: "2026-07-02T21:41:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:41:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:50Z" name: success-200-isvc-1702f-predictor.18be96f4d174d905 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 25a009f6-e9a9-4d57-a482-b821c5a6c223 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:41:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1702f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: b172e143-ef2d-4de5-b632-e79816240026 kind: Event lastTimestamp: "2026-07-02T21:41:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:41:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:50Z" name: success-200-isvc-1702f-predictor.18be96f4d1d3f384 namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: bf0f3c63-4e75-4096-aab7-b463d5e28a13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1702f namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: 28f97302-6be5-4138-b8ba-10f1545fccc0 kind: Event lastTimestamp: "2026-07-02T21:41:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:39Z" name: success-200-isvc-1702f.18be96edd3a0c4b4 namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: abdbadee-e9d3-45a9-89dd-8287209ebc96 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1702f namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: 28f97302-6be5-4138-b8ba-10f1545fccc0 kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-1702f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1702f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:50Z" name: success-200-isvc-1702f.18be96edd477c694 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 0dd7ca01-fbe7-496e-b8b0-370ac71f4357 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1702f namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: 28f97302-6be5-4138-b8ba-10f1545fccc0 kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1702f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:50Z" name: success-200-isvc-1702f.18be96edd4782d40 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 6ce4d8da-45c0-482f-8a59-a3f32822cc6b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1702f namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 28f97302-6be5-4138-b8ba-10f1545fccc0 kind: Event lastTimestamp: "2026-07-02T21:41:39Z" message: InferenceService [success-200-isvc-1702f] is Ready metadata: creationTimestamp: "2026-07-02T21:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:39Z" name: success-200-isvc-1702f.18be96f9441b7810 namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 0d5062e7-0d1b-4ceb-b006-ce9c0f64edd5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:52:00.267955Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be9789b9bfcce8 namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: 3544cf46-3244-41b1-98ba-073d6362fdb4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be9789d37fc08c namespace: kserve-ci-e2e-test resourceVersion: "26600" uid: 5037bee0-dd14-42a5-adde-4c4db0f31727 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be9789d4c05165 namespace: kserve-ci-e2e-test resourceVersion: "26602" uid: b1384e7f-c11d-480b-8009-f0b40f5f7998 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be9789d83110b0 namespace: kserve-ci-e2e-test resourceVersion: "26604" uid: ada956b9-badf-4244-923c-17673c6d75d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be9789d8a1d142 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: d64faeda-f257-4d34-bd10-0fc0a7835fd8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be9789d8ab8f14 namespace: kserve-ci-e2e-test resourceVersion: "26606" uid: 57d4bb9b-bb5b-494a-b057-75737508f32e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be9789dba495c3 namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: a6e4247d-06e7-40c9-b87e-df1de41d4e80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be9789dc176b00 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 5b195fd3-8d2b-4b90-80f3-68ae0c965a58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:52:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T21:52:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:52:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:38Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be978a3cfdde43 namespace: kserve-ci-e2e-test resourceVersion: "26992" uid: 182e2116-9be4-42f8-8e8b-4cf2e89f6cb9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T22:01:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T22:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:01:15Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be980ae6ff958d namespace: kserve-ci-e2e-test resourceVersion: "30654" uid: 18867c5c-da0a-4c45-b32a-0e9196285333 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 53cf0706-89f4-4e4f-90fa-f798c6c55398 kind: Event lastTimestamp: "2026-07-02T22:01:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T22:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:01:15Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88.18be980ae7000c44 namespace: kserve-ci-e2e-test resourceVersion: "30655" uid: 33ae9bad-aeb9-42f7-9d6e-6552bc18d149 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-67ac0-predictor-6f7fc66ddb namespace: kserve-ci-e2e-test resourceVersion: "26531" uid: f555b054-d281-4fe2-b680-d0541900e3a8 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: 'Created pod: success-200-isvc-67ac0-predictor-6f7fc66ddb-54h88' metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor-6f7fc66ddb.18be9789b95501d6 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 7f00400f-7cf9-4448-80ea-6d7a0c0e8776 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: f026bc95-ce59-473d-9909-23c543e5e83c kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Scaled up replica set success-200-isvc-67ac0-predictor-6f7fc66ddb from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-67ac0-predictor.18be9789b77f3e31 namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: a0cd1342-4183-4656-a606-8cc97d8147b5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:52:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: 6ef775e6-c97f-4332-acf0-fb31d4e441e8 kind: Event lastTimestamp: "2026-07-02T21:52:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:30Z" name: success-200-isvc-67ac0-predictor.18be978d3602cc8d namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: c3f58143-e41c-4861-aaf7-43f901a8b7b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:52:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: 6ef775e6-c97f-4332-acf0-fb31d4e441e8 kind: Event lastTimestamp: "2026-07-02T21:52:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:30Z" name: success-200-isvc-67ac0-predictor.18be978d366bfe7b namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 6576d917-8c6d-4cc1-83fe-6ee951c2de40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:52:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 6ef775e6-c97f-4332-acf0-fb31d4e441e8 kind: Event lastTimestamp: "2026-07-02T21:53:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:53:15Z" name: success-200-isvc-67ac0-predictor.18be97943465d89a namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: a9bd6281-602f-4852-9ac7-abf6ea4cc223 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:52:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67ac0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 6ef775e6-c97f-4332-acf0-fb31d4e441e8 kind: Event lastTimestamp: "2026-07-02T21:53:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:53:15Z" name: success-200-isvc-67ac0-predictor.18be979434ca9666 namespace: kserve-ci-e2e-test resourceVersion: "27315" uid: 6bc7426e-f4f8-4c2d-9e19-c484f042e395 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: 264e0d6b-b701-4636-b6f2-d00948de0763 kind: Event lastTimestamp: "2026-07-02T21:52:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:48Z" name: success-200-isvc-67ac0.18be9789b7b2d86e namespace: kserve-ci-e2e-test resourceVersion: "27084" uid: 7b7fd3d8-7c8e-472a-93b2-f94aefd91ab7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67ac0 namespace: kserve-ci-e2e-test resourceVersion: "27082" uid: 264e0d6b-b701-4636-b6f2-d00948de0763 kind: Event lastTimestamp: "2026-07-02T21:52:48Z" message: InferenceService [success-200-isvc-67ac0] is Ready metadata: creationTimestamp: "2026-07-02T21:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:48Z" name: success-200-isvc-67ac0.18be9794f7bd3556 namespace: kserve-ci-e2e-test resourceVersion: "27083" uid: 146d93cc-261d-44cf-9cb5-163535ce2f90 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:31:10.053196Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7615e-predictor-6fc7996664-4pfhn to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be9666a32472ae namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: 51089a8c-d72f-49bd-98d6-4ef0c742a821 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be9666bcb31fdc namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 1ee743e5-5ec1-4019-9d60-be7ee8ed0d39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be9666bdea8e42 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: e28d110c-fe33-4354-aa24-ee16e9246058 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be9666c1019078 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: ac0c3f74-c064-4157-b393-c72daa46378d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be9666c1631851 namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: bc1cf147-9f20-466c-8085-41f6e4b78867 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be9666c16d1a08 namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 8c527536-3832-469b-980f-62ac7ae7c011 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be9666c4978f7c namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 2b3562e5-4525-4b2f-8713-31bf4363c639 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be9666c4fc7698 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: e6f545d6-cf07-4fe4-9ab8-39e2ad980e03 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:40:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:27Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be9666eccc5faf namespace: kserve-ci-e2e-test resourceVersion: "20225" uid: 59b48e64-413a-45dc-a77c-8f1a459e03df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:40:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:24Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be96e7ce3fbb67 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 40c24e76-b021-4af4-ae98-97ebea3136d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:40:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:24Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be96e7ce4005b8 namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: 349b8638-c0b6-4f27-9ced-c1382f9573c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 13a08355-135a-43a9-859d-b69891f40231 kind: Event lastTimestamp: "2026-07-02T21:40:27Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:27Z" name: success-200-isvc-7615e-predictor-6fc7996664-4pfhn.18be96e8613141e0 namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: ed583cc3-336d-4d8b-abaa-7f01e649c883 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7615e-predictor-6fc7996664 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: d93489a4-99a8-4cd9-a1af-5ca8177ea44c kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: 'Created pod: success-200-isvc-7615e-predictor-6fc7996664-4pfhn' metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e-predictor-6fc7996664.18be9666a2a926b3 namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: 1fb52ead-abed-42f9-95e2-a5ce8bb44a3a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: fc679cfe-b2a8-46e1-a44b-da47567f8e56 kind: Event lastTimestamp: "2026-07-02T21:31:09Z" message: Scaled up replica set success-200-isvc-7615e-predictor-6fc7996664 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:09Z" name: success-200-isvc-7615e-predictor.18be96669f6b53d6 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: 727fad99-14be-490f-a48d-1402883717db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:31:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 10126829-2560-4517-97b5-38027201d746 kind: Event lastTimestamp: "2026-07-02T21:31:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:55Z" name: success-200-isvc-7615e-predictor.18be966a1e5c0170 namespace: kserve-ci-e2e-test resourceVersion: "16433" uid: 4c57b24d-897a-48dd-9a95-a6ba28efa9a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:31:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 10126829-2560-4517-97b5-38027201d746 kind: Event lastTimestamp: "2026-07-02T21:31:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:55Z" name: success-200-isvc-7615e-predictor.18be966a1ec2e624 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: a98e53ad-d124-4d9b-a355-6395b18d1af8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:31:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 10126829-2560-4517-97b5-38027201d746 kind: Event lastTimestamp: "2026-07-02T21:32:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:25Z" name: success-200-isvc-7615e-predictor.18be966d9d73a47b namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: e1697c52-bb36-40f1-a29d-7cd3a2937324 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:31:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7615e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 10126829-2560-4517-97b5-38027201d746 kind: Event lastTimestamp: "2026-07-02T21:32:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:25Z" name: success-200-isvc-7615e-predictor.18be966d9dd96220 namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: 9fc36772-8c45-45d1-9504-6aefa7d49bfa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:31:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7615e namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 92bcb03a-8246-4384-9822-b927571366c1 kind: Event lastTimestamp: "2026-07-02T21:31:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:57Z" name: success-200-isvc-7615e.18be96669f6ef188 namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: a8eef9d7-1472-4214-86f0-6e056f31d1cf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7615e namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 92bcb03a-8246-4384-9822-b927571366c1 kind: Event lastTimestamp: "2026-07-02T21:31:09Z" message: 'Failed to update status for InferenceService "success-200-isvc-7615e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7615e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:09Z" name: success-200-isvc-7615e.18be96669fdce4c0 namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 5d1207d9-0551-4643-b0e1-83b41fe07334 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7615e namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 92bcb03a-8246-4384-9822-b927571366c1 kind: Event lastTimestamp: "2026-07-02T21:31:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7615e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-7615e.18be96669fdd44c6 namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: e42c1a5e-83c8-4e62-a139-fa4407802612 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7615e namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: 92bcb03a-8246-4384-9822-b927571366c1 kind: Event lastTimestamp: "2026-07-02T21:31:57Z" message: InferenceService [success-200-isvc-7615e] is Ready metadata: creationTimestamp: "2026-07-02T21:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:31:57Z" name: success-200-isvc-7615e.18be9671a7bd9b71 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: adcc1e1e-576f-4d38-baa4-3321b5106f1c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:51:15.899818Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25970" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:51:15Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977f6533b911 namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: 464fcb41-1ce4-449d-9a25-7d52e77150bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T21:51:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-76a5f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:16Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977f7068ebaf namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: 74028e6b-d5d5-4ba8-b9c6-d37655d1d8ae reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25979" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T21:51:16Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:51:16Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977fa2814cf1 namespace: kserve-ci-e2e-test resourceVersion: "26044" uid: 827ca39b-3029-46c7-a17a-9fcb8bb3f03e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T21:51:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:16Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977fa4385155 namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 001d0dda-86d3-4a28-a9f4-2cdf6a3cf9fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977fa7080794 namespace: kserve-ci-e2e-test resourceVersion: "26048" uid: c6568b37-6ad3-4cfe-b722-8689487a428d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977fa7776b43 namespace: kserve-ci-e2e-test resourceVersion: "26049" uid: 095a0206-a4fe-4a1d-a8cd-2a819bc6612b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977fa781dc4c namespace: kserve-ci-e2e-test resourceVersion: "26051" uid: 1f9db4ec-c515-408d-8c6e-ab4ca2fb12c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977faa93b1a5 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 278cf27c-b0c8-4821-b149-54a15cfdb9e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T21:51:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:17Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977fab048d6b namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 5a7ddda2-29bd-4e4a-9dc5-8ae5d9f2f996 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:51:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T21:51:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:54Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be977ff33e9c2b namespace: kserve-ci-e2e-test resourceVersion: "26439" uid: b1e80df9-5266-4a7b-b9c8-2f64dd558369 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T22:08:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T22:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:08:35Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be98716b04ba43 namespace: kserve-ci-e2e-test resourceVersion: "33699" uid: 41da2f2e-c3a5-416b-aca5-e0172a22cdea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 8bfa0b5e-ac46-4503-a2d0-c0a679de6543 kind: Event lastTimestamp: "2026-07-02T22:08:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T22:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:08:35Z" name: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5.18be98716b05427e namespace: kserve-ci-e2e-test resourceVersion: "33700" uid: 4be8cced-fb36-4a49-a854-b40438a3219a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-76a5f-predictor-68958f7c46 namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: ded77c85-6144-4f18-a181-723dcf44d0ce kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: 'Created pod: success-200-isvc-76a5f-predictor-68958f7c46-8ddl5' metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:15Z" name: success-200-isvc-76a5f-predictor-68958f7c46.18be977f64934799 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: a3ca21a4-5865-4f20-8577-67ae13f26e77 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25946" uid: 735e5bb6-c41d-426b-a4a7-ecda27fd7b78 kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: Scaled up replica set success-200-isvc-76a5f-predictor-68958f7c46 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:15Z" name: success-200-isvc-76a5f-predictor.18be977f61d430f8 namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: bc20aea6-241e-4331-aa1f-698dcbcdc0b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: 3192275c-917b-4418-bf58-6d1926a22978 kind: Event lastTimestamp: "2026-07-02T21:51:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:30Z" name: success-200-isvc-76a5f-predictor.18be9782e0537388 namespace: kserve-ci-e2e-test resourceVersion: "26191" uid: b71aa21e-a856-493d-bd79-37780be959d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: 3192275c-917b-4418-bf58-6d1926a22978 kind: Event lastTimestamp: "2026-07-02T21:51:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:30Z" name: success-200-isvc-76a5f-predictor.18be9782e0bce297 namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: 06444b7b-ad5d-456d-b3dd-ad6e50daebe3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:51:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: 3192275c-917b-4418-bf58-6d1926a22978 kind: Event lastTimestamp: "2026-07-02T21:52:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:51:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:15Z" name: success-200-isvc-76a5f-predictor.18be97865f8c71d9 namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: 58d6de08-04f7-4d54-866b-642f90d342c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:51:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-76a5f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: 3192275c-917b-4418-bf58-6d1926a22978 kind: Event lastTimestamp: "2026-07-02T21:52:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:51:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:15Z" name: success-200-isvc-76a5f-predictor.18be97865ff53895 namespace: kserve-ci-e2e-test resourceVersion: "26775" uid: fbd72baa-c913-42db-b826-884a571b2a3f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-76a5f namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: 391cbb49-4193-478e-9800-fe0f26d52277 kind: Event lastTimestamp: "2026-07-02T21:52:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:04Z" name: success-200-isvc-76a5f.18be977f61ea671b namespace: kserve-ci-e2e-test resourceVersion: "26685" uid: 26341b92-8709-4c37-bf09-5d83be993c39 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-76a5f namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: 391cbb49-4193-478e-9800-fe0f26d52277 kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-76a5f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-76a5f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:51:15Z" name: success-200-isvc-76a5f.18be977f62d8be0b namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: 48a86597-f97d-4095-9218-37cf80004a32 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-76a5f namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: 391cbb49-4193-478e-9800-fe0f26d52277 kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-76a5f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:51:15Z" name: success-200-isvc-76a5f.18be977f62d908f4 namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: cd9e7dc4-3129-48d3-aef4-9e2e81e30a5f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-76a5f namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 391cbb49-4193-478e-9800-fe0f26d52277 kind: Event lastTimestamp: "2026-07-02T21:52:04Z" message: InferenceService [success-200-isvc-76a5f] is Ready metadata: creationTimestamp: "2026-07-02T21:52:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:04Z" name: success-200-isvc-76a5f.18be978aae03459c namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: d70d21da-8ec8-4dc4-9417-3fce1df04d8b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:41:35.099289Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-87dcc-predictor-648644cd6-hvcgd to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be96f82acaf82e namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: fbea4114-b66d-420a-a767-467f500d6201 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be96f84424097b namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: d397f222-7cd0-48f5-86ae-f2d259731f23 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be96f845b3cbd6 namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: adde9c01-bfc5-47d6-8281-4c502cca7b79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be96f8488086e7 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: bb8f6d6f-e2ce-4a4a-b2d5-e85e26bd3bff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be96f848f77e37 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 84d174b1-b255-4388-a6de-5b220300cb9d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be96f849025a97 namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: cbd506af-6d1d-411a-80d4-52c3423da51f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be96f84bfecac1 namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: ffed5937-a83e-412f-a85d-8c86a927bafa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be96f84c6cc050 namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: 72295375-b8ae-4f48-89b2-5bdd5851d7b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be96f85a8e6edc namespace: kserve-ci-e2e-test resourceVersion: "25732" uid: 4b09ca3e-6671-4340-9e3f-a82dee702f05 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:50Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be97795eb53824 namespace: kserve-ci-e2e-test resourceVersion: "25640" uid: b0a9dd65-f5dc-450d-b2c9-87d3d3cd2ee4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:50Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be97795eb6bb41 namespace: kserve-ci-e2e-test resourceVersion: "25644" uid: acaf73cc-28a5-4aa2-bc7f-10d3f311b63c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: fd57752b-4d3f-4887-9433-f69b266be07c kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: success-200-isvc-87dcc-predictor-648644cd6-hvcgd.18be9779ceeb2098 namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: bccda695-c05b-42ae-9ad1-29d7c4366ab8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-87dcc-predictor-648644cd6 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 5fdf2e54-9f75-46eb-b379-b95d0dc99f60 kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: 'Created pod: success-200-isvc-87dcc-predictor-648644cd6-hvcgd' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor-648644cd6.18be96f82a2ea7e0 namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 7c4f271f-b982-4571-bf3f-68e534795035 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: bba27ec9-b5c4-48f6-9f27-66de1078105d kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Scaled up replica set success-200-isvc-87dcc-predictor-648644cd6 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc-predictor.18be96f8286d125d namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: b3f4af27-f182-4c25-84d5-b1533f2c30b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:41:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 616fd980-7546-4927-8a35-efb9d0e72969 kind: Event lastTimestamp: "2026-07-02T21:42:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:20Z" name: success-200-isvc-87dcc-predictor.18be96fba7478b3a namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: a7af8e96-c9de-42a8-8d25-1dc6074d088e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:41:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 616fd980-7546-4927-8a35-efb9d0e72969 kind: Event lastTimestamp: "2026-07-02T21:42:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:20Z" name: success-200-isvc-87dcc-predictor.18be96fba7a41daa namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: 37885be5-68a5-4881-a26f-30782a1f1256 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 616fd980-7546-4927-8a35-efb9d0e72969 kind: Event lastTimestamp: "2026-07-02T21:42:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:50Z" name: success-200-isvc-87dcc-predictor.18be970624499322 namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 17303946-583c-429a-8b0f-73fc31fc1abb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87dcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 616fd980-7546-4927-8a35-efb9d0e72969 kind: Event lastTimestamp: "2026-07-02T21:42:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:50Z" name: success-200-isvc-87dcc-predictor.18be970624ac21b4 namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: a4acc407-12ac-40b2-8447-43a611541b6b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87dcc namespace: kserve-ci-e2e-test resourceVersion: "20984" uid: 378a43df-07eb-4172-8868-4a6b0fcf2ff8 kind: Event lastTimestamp: "2026-07-02T21:42:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:22Z" name: success-200-isvc-87dcc.18be96f82895c1d9 namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: 5ea6a3f1-92b0-4c46-aac4-ff7dfc6bb7b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87dcc namespace: kserve-ci-e2e-test resourceVersion: "20984" uid: 378a43df-07eb-4172-8868-4a6b0fcf2ff8 kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: 'Failed to update status for InferenceService "success-200-isvc-87dcc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-87dcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc.18be96f828fedff3 namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: f66bef88-a7f1-412d-81e5-c36158997471 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87dcc namespace: kserve-ci-e2e-test resourceVersion: "20984" uid: 378a43df-07eb-4172-8868-4a6b0fcf2ff8 kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-87dcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-87dcc.18be96f828ff3bdf namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: d2da6072-e04e-43e9-bef2-ce44274a2bb3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87dcc namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: 378a43df-07eb-4172-8868-4a6b0fcf2ff8 kind: Event lastTimestamp: "2026-07-02T21:42:22Z" message: InferenceService [success-200-isvc-87dcc] is Ready metadata: creationTimestamp: "2026-07-02T21:42:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:22Z" name: success-200-isvc-87dcc.18be9703167d8cfd namespace: kserve-ci-e2e-test resourceVersion: "21696" uid: 965e5ae8-7e4a-412f-b16b-7290ed356c57 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:42:00.975960Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8bf58-predictor-695db95966-9w5pz to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:42:00Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be96fe3129a445 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: db4cf644-a9b3-4f7c-b50c-a2435d5d5d69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:42:01Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be96fe4a96e75b namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: f4c6d469-a523-4327-bb53-81211ad3302e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be96fe4bbf7d4f namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: 5c960c0e-271a-4fdc-bfcd-97cac47335c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be96fe4ef2ccdd namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: be3f656d-d4a5-4701-b5ee-7daa0562bd67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be96fe4f5f025f namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: f96ef6f2-d1a3-4085-a0ae-9e5e7d9ba8ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be96fe4f7c1d01 namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: 1a9ac2df-10a7-4972-946c-dba2d5df0ce1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be96fe5242cdbb namespace: kserve-ci-e2e-test resourceVersion: "21497" uid: ec46d309-d9e7-48e6-9ef7-bb15d4801692 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:42:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:01Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be96fe52b945bb namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: c71f9a3a-aeb0-48f7-a570-ecaa90ca23b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:42:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:51:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:18Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be96fe6f562b0a namespace: kserve-ci-e2e-test resourceVersion: "26076" uid: 9f2d9583-45b5-4048-adcf-68e93f8391c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:15Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be977f63aeb0a6 namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: 3431eb67-61b8-426e-82a1-7fc150f23505 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:51:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:15Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be977f63afce5b namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: 66959211-2a9c-438f-b4cf-13d5848a8f6f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bf58-predictor-695db95966-9w5pz namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 6ae0814a-bb2b-4049-8980-2e1654c9dcda kind: Event lastTimestamp: "2026-07-02T21:51:18Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:18Z" name: success-200-isvc-8bf58-predictor-695db95966-9w5pz.18be977fe3d3b1eb namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: d1e35328-900b-44ce-9b61-5a99e0eb395f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8bf58-predictor-695db95966 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: fc9b82f9-cfb5-4d6f-b1a7-1781ef9a3f24 kind: Event lastTimestamp: "2026-07-02T21:42:00Z" message: 'Created pod: success-200-isvc-8bf58-predictor-695db95966-9w5pz' metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:00Z" name: success-200-isvc-8bf58-predictor-695db95966.18be96fe309f2123 namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: a5585767-e558-4056-ac44-73bf831bd3e5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: a2d79566-9af9-490a-a1e8-51e3a8981f03 kind: Event lastTimestamp: "2026-07-02T21:42:00Z" message: Scaled up replica set success-200-isvc-8bf58-predictor-695db95966 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:00Z" name: success-200-isvc-8bf58-predictor.18be96fe2e5beeb6 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: ba40d1a2-d2a1-4345-9f6c-15c17e73e154 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: ea5b2ead-9d9a-415d-8486-0e79703032ec kind: Event lastTimestamp: "2026-07-02T21:42:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:30Z" name: success-200-isvc-8bf58-predictor.18be9701ace07ac3 namespace: kserve-ci-e2e-test resourceVersion: "21776" uid: 8eb7f2c3-f905-4d93-ae84-ae7897834125 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: ea5b2ead-9d9a-415d-8486-0e79703032ec kind: Event lastTimestamp: "2026-07-02T21:42:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:30Z" name: success-200-isvc-8bf58-predictor.18be9701ad473f15 namespace: kserve-ci-e2e-test resourceVersion: "21777" uid: e98b026b-9f8a-4f8e-942e-56f7589dfa19 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: ea5b2ead-9d9a-415d-8486-0e79703032ec kind: Event lastTimestamp: "2026-07-02T21:43:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:00Z" name: success-200-isvc-8bf58-predictor.18be9708aac5a938 namespace: kserve-ci-e2e-test resourceVersion: "22066" uid: 7044323e-5b0c-4a2a-a20f-95d0a2cd9ff7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8bf58-predictor namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: ea5b2ead-9d9a-415d-8486-0e79703032ec kind: Event lastTimestamp: "2026-07-02T21:43:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:00Z" name: success-200-isvc-8bf58-predictor.18be9708ab2b0354 namespace: kserve-ci-e2e-test resourceVersion: "22069" uid: 71436ae5-49bb-4b0e-957e-17fa1a363af6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: 2e82046c-2dbe-40cc-939b-f487902262b7 kind: Event lastTimestamp: "2026-07-02T21:42:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:48Z" name: success-200-isvc-8bf58.18be96fe2e62b509 namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: 364b6c81-f186-4dd3-a4ee-1e7d515c5341 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: 2e82046c-2dbe-40cc-939b-f487902262b7 kind: Event lastTimestamp: "2026-07-02T21:42:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-8bf58": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8bf58": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:00Z" name: success-200-isvc-8bf58.18be96fe2f245084 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: 6dfb6d42-9987-436f-9fb1-3d47aeb522a5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: 2e82046c-2dbe-40cc-939b-f487902262b7 kind: Event lastTimestamp: "2026-07-02T21:42:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8bf58": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:00Z" name: success-200-isvc-8bf58.18be96fe2f24a870 namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 6d2d7def-5236-46a6-b290-27cbf630e398 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8bf58 namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 2e82046c-2dbe-40cc-939b-f487902262b7 kind: Event lastTimestamp: "2026-07-02T21:42:48Z" message: InferenceService [success-200-isvc-8bf58] is Ready metadata: creationTimestamp: "2026-07-02T21:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:48Z" name: success-200-isvc-8bf58.18be97092a9d0217 namespace: kserve-ci-e2e-test resourceVersion: "21955" uid: 1a8737a3-b8d9-42ac-a147-28f693952356 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:31:35.840403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:31:35Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966ca42e07d8 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 645ce5a7-6e9e-4cbb-8d31-0aa051ba0298 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a3b0c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966cb1102723 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: 543545ad-fec8-4440-982f-e6346d9255e8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:31:36Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966ce1e06d33 namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: b014de08-219d-483b-872b-812c3c163e58 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966ce3749431 namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: b367b400-6554-44a6-b9ee-f50f3b1344b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966ce6e1670b namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: bf768a4a-2e35-43fb-829d-f312145e1aa4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966ce7441e32 namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: ca37427a-fae4-4550-b3aa-f966a9ac7549 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966ce74e9f87 namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: 5e810c72-d3ea-4f00-a33e-a61c30e862e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:31:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:37Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966cebbbd97a namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: 73ad5b3a-1bca-4441-bf34-7f7576128dba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:31:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:37Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966cec221338 namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: 3b3835ba-b666-40f4-bbac-75035dd9ee19 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:40:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:53Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be966d01ffd819 namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: 0f1ab9b8-607f-4103-9f3b-006330590eb1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:50Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be96edd5d1029e namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 38751c24-923b-4275-a462-0310e4af7b98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:40:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:50Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be96edd5d17aa8 namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 15b72fbc-c73e-4715-8a5e-b1b99fb474ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 83ed67a9-3f3d-4452-a21e-65b6eb3913ec kind: Event lastTimestamp: "2026-07-02T21:40:53Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:53Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn.18be96ee766de4ed namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: 5015dcbf-54b4-47b4-95e2-db84bac19c4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a3b0c-predictor-7ccb86bd58 namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 64a52d0f-da0c-46f0-91a8-28af85485442 kind: Event lastTimestamp: "2026-07-02T21:31:35Z" message: 'Created pod: success-200-isvc-a3b0c-predictor-7ccb86bd58-qfxvn' metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:35Z" name: success-200-isvc-a3b0c-predictor-7ccb86bd58.18be966ca3c5c8be namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 8e13aec7-6fb0-4e6d-8fd6-f9d00cdc4d15 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: a9ab2157-8106-4fbc-9138-0cc43028a984 kind: Event lastTimestamp: "2026-07-02T21:31:35Z" message: Scaled up replica set success-200-isvc-a3b0c-predictor-7ccb86bd58 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:35Z" name: success-200-isvc-a3b0c-predictor.18be966ca1d2da46 namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: 3875b52d-4d3d-45c7-96b7-9b254fd5befb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:31:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 707888a8-4f26-4869-83d0-6955dc31c2e9 kind: Event lastTimestamp: "2026-07-02T21:32:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:05Z" name: success-200-isvc-a3b0c-predictor.18be967020afa247 namespace: kserve-ci-e2e-test resourceVersion: "16534" uid: 909a4773-3429-48b9-85c6-8434fc7f7ae3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:31:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 707888a8-4f26-4869-83d0-6955dc31c2e9 kind: Event lastTimestamp: "2026-07-02T21:32:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:05Z" name: success-200-isvc-a3b0c-predictor.18be96702110c973 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: 7a6d9f9a-4fe5-4e3d-9edb-51144e1f21b9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:32:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 707888a8-4f26-4869-83d0-6955dc31c2e9 kind: Event lastTimestamp: "2026-07-02T21:32:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:50Z" name: success-200-isvc-a3b0c-predictor.18be96771eee6f0e namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 859dcaee-9cf0-4437-9660-3e7a0338f586 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:32:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3b0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 707888a8-4f26-4869-83d0-6955dc31c2e9 kind: Event lastTimestamp: "2026-07-02T21:32:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:50Z" name: success-200-isvc-a3b0c-predictor.18be96771f59c4b0 namespace: kserve-ci-e2e-test resourceVersion: "16989" uid: d9784f83-5502-4b9d-bc30-c86874fdaf10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:31:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 99422eff-b336-42fe-be10-cabaf37a34f1 kind: Event lastTimestamp: "2026-07-02T21:32:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:31:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:23Z" name: success-200-isvc-a3b0c.18be966ca1d812d1 namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: 39afcca3-d629-4cad-80a0-5f6f724d4bf3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3b0c namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: 99422eff-b336-42fe-be10-cabaf37a34f1 kind: Event lastTimestamp: "2026-07-02T21:32:23Z" message: InferenceService [success-200-isvc-a3b0c] is Ready metadata: creationTimestamp: "2026-07-02T21:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:23Z" name: success-200-isvc-a3b0c.18be9677be76bafb namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: bafee064-6762-4345-9c19-b0a55f305056 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:50:50.008609Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25624" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:50:50Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be97795df767b6 namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: e9ad2bd0-6174-4172-9be6-64d8fa36feb9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c9d16-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:50Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be97796d8276bd namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: dd77e0e1-6dca-4b17-9a0f-e28023bb9d3a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:50:51Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be97799b64e91f namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: bb14ef6b-9a9b-4ff4-a8ef-edebf0fc3793 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be97799cbe7008 namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: a0a6c7d0-48ca-45d1-8346-41a3a1ef605f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be9779a0e1e46e namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 6140ebfb-f43f-464d-a71d-aad44ed5e63d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be9779a15397f0 namespace: kserve-ci-e2e-test resourceVersion: "25712" uid: f56b88ef-ded5-45b3-9d9c-46a0a5b14b78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be9779a15f12e4 namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 6a6e4622-e223-4897-92d6-c1405f3310f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be9779a4948356 namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: 774c1d0c-c7cc-48f6-bd3a-589f7ec7b268 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:50:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:50:51Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be9779a502c8f7 namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 16dbb466-efce-4f05-9f4b-4b3a06abc733 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:51:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:51:28Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be9779debd77e3 namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: 0db46849-48ae-4cec-b140-325d6b8e6c28 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be9789b93559bd namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 9f4cd49f-2e87-4097-ba46-b9bfdb773c97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:52:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:00Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be9789b935e9a6 namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: e379c872-e82e-4da0-a019-2968a4b18fb0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 7c34c887-38b7-4dcd-8043-c726de59f2f7 kind: Event lastTimestamp: "2026-07-02T21:52:03Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:52:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:03Z" name: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk.18be978a66fa715c namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 3217d5bb-02fe-41bb-95f6-217414cc8b34 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c9d16-predictor-664b8c5c66 namespace: kserve-ci-e2e-test resourceVersion: "25607" uid: 83680d4e-244b-42f4-b530-fb410217c673 kind: Event lastTimestamp: "2026-07-02T21:50:50Z" message: 'Created pod: success-200-isvc-c9d16-predictor-664b8c5c66-7g9zk' metadata: creationTimestamp: "2026-07-02T21:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:50:50Z" name: success-200-isvc-c9d16-predictor-664b8c5c66.18be97795d92f4e1 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: 0f6c37c1-a193-4ca6-93a3-fc332de5caff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: 021e0881-0d88-4833-a3c5-8d498badb6d5 kind: Event lastTimestamp: "2026-07-02T21:50:49Z" message: Scaled up replica set success-200-isvc-c9d16-predictor-664b8c5c66 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:50:49Z" name: success-200-isvc-c9d16-predictor.18be977959f3f070 namespace: kserve-ci-e2e-test resourceVersion: "25609" uid: 9743aaa0-e85c-4e92-9d75-ae45885805e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: eb79e581-3908-4f08-b933-833d07db5e00 kind: Event lastTimestamp: "2026-07-02T21:51:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:51:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:19Z" name: success-200-isvc-c9d16-predictor.18be977cd85b9bee namespace: kserve-ci-e2e-test resourceVersion: "26102" uid: 68eb9548-7979-4a1b-b338-09c7e37ad985 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: eb79e581-3908-4f08-b933-833d07db5e00 kind: Event lastTimestamp: "2026-07-02T21:51:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:51:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:19Z" name: success-200-isvc-c9d16-predictor.18be977cd8a0d827 namespace: kserve-ci-e2e-test resourceVersion: "26103" uid: 43423511-4030-44ee-9f86-543c94745a8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: eb79e581-3908-4f08-b933-833d07db5e00 kind: Event lastTimestamp: "2026-07-02T21:51:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:50Z" name: success-200-isvc-c9d16-predictor.18be9783d66345fd namespace: kserve-ci-e2e-test resourceVersion: "26370" uid: 91f9b748-0d11-4f50-b38a-24d8d4f6d29a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:51:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c9d16-predictor namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: eb79e581-3908-4f08-b933-833d07db5e00 kind: Event lastTimestamp: "2026-07-02T21:51:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:51:50Z" name: success-200-isvc-c9d16-predictor.18be9783d6cc798e namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: 852c3b61-acee-4364-ae69-d9bf92cfd380 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:50:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "25597" uid: eef11fb0-8c9f-4788-aa0a-c9630a254a4d kind: Event lastTimestamp: "2026-07-02T21:51:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:51:38Z" name: success-200-isvc-c9d16.18be977959f04402 namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: 69fc03da-febd-4d70-9866-e2c4155a8bdd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "25597" uid: eef11fb0-8c9f-4788-aa0a-c9630a254a4d kind: Event lastTimestamp: "2026-07-02T21:50:49Z" message: 'Failed to update status for InferenceService "success-200-isvc-c9d16": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c9d16": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:50:49Z" name: success-200-isvc-c9d16.18be97795a8c0653 namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: 782bc260-9cd3-4e8b-9eb5-26fd74ad939c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:50:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "25597" uid: eef11fb0-8c9f-4788-aa0a-c9630a254a4d kind: Event lastTimestamp: "2026-07-02T21:50:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c9d16": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:50:49Z" name: success-200-isvc-c9d16.18be97795a8c6bd8 namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: 507b0ca6-2468-4b68-b27e-dbbb2ef2a653 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:51:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c9d16 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: eef11fb0-8c9f-4788-aa0a-c9630a254a4d kind: Event lastTimestamp: "2026-07-02T21:51:38Z" message: InferenceService [success-200-isvc-c9d16] is Ready metadata: creationTimestamp: "2026-07-02T21:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:51:38Z" name: success-200-isvc-c9d16.18be9784998d4ceb namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: 29cacf93-b51b-41fb-9131-eba4be2eed1e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:29:25.510101Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:29:25Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be964e4be37f78 namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: afb8340b-0db0-41b8-83c0-d395b874e3d7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:29:25Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:29:25Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be964e659c51b4 namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: 06ad14b9-ed21-4942-99ee-399a467d0c7d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:29:25Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:25Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be964e66e68bf2 namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: eec9af18-aa93-4a1a-a9f4-6381913cea11 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" in 15.827s (15.827s including waiting). Image size: 1335616100 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be965216493f8d namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 6baeaf80-575b-4892-b84d-f1edc7aec531 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be96521b6d3dc9 namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: f41d5179-5c7f-4408-a765-c77c9816eb23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be96521c5bcd19 namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 50d2dff0-0da4-49cd-a086-14ce1fffd4fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T21:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:41Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be96521c6c9e7a namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: c76101c0-99ca-482e-a7f1-5c85a703b112 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:29:46Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.994s (4.994s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:46Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be965346245b7b namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: de8db1e5-2ba9-4431-99b5-a2bb02b03aff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:29:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:47Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be96534f114587 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: a9245158-d747-4e12-86ca-5c9c7ac7ab65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:29:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:47Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be96535011110e namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 3f449b0e-6f3b-42f5-a749-be4e7442963c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:30:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:33Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be965382aed763 namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 629fb111-a2de-4291-9f4d-1528eb9a22ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be9666a503a016 namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: c72113b6-3800-40d9-81c6-cb6928939410 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 2f5229d4-257f-4fd1-9593-85e274c1be8a kind: Event lastTimestamp: "2026-07-02T21:31:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:10Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs.18be9666a5045097 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 80443b91-d38d-4802-9359-ab6667b2f0b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e36cd-predictor-fdf4cf55d namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: ce1301da-c26f-4bf3-bfcd-579da6e66eb8 kind: Event lastTimestamp: "2026-07-02T21:29:25Z" message: 'Created pod: success-200-isvc-e36cd-predictor-fdf4cf55d-dfmzs' metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:25Z" name: success-200-isvc-e36cd-predictor-fdf4cf55d.18be964e4b57ac0c namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: eae16d76-d81e-4a62-b64c-8d3aacfce154 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: cfcbd0bd-b4c2-4dd7-9d34-ec35c131c3fa kind: Event lastTimestamp: "2026-07-02T21:29:25Z" message: Scaled up replica set success-200-isvc-e36cd-predictor-fdf4cf55d from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:25Z" name: success-200-isvc-e36cd-predictor.18be964e47a52b88 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: b65a33f1-2ee5-4ff4-9a9a-383221b37342 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: 320b380b-2f70-4ca0-b05d-aedf7cfeda35 kind: Event lastTimestamp: "2026-07-02T21:30:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:25Z" name: success-200-isvc-e36cd-predictor.18be9651cd1f1edd namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: 62048dff-cbe4-49bb-8ee7-72308c6a4234 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: 320b380b-2f70-4ca0-b05d-aedf7cfeda35 kind: Event lastTimestamp: "2026-07-02T21:30:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:25Z" name: success-200-isvc-e36cd-predictor.18be9651cd7f3f12 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 35fd96bd-dc68-4d6e-971e-876a607c7272 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:30:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 320b380b-2f70-4ca0-b05d-aedf7cfeda35 kind: Event lastTimestamp: "2026-07-02T21:30:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:55Z" name: success-200-isvc-e36cd-predictor.18be9658cc7e984b namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: d907bf94-9961-41dd-96f9-3487e20e5dad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:30:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e36cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 320b380b-2f70-4ca0-b05d-aedf7cfeda35 kind: Event lastTimestamp: "2026-07-02T21:30:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:55Z" name: success-200-isvc-e36cd-predictor.18be9658ccf16043 namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 86aa2a17-09d3-4238-b231-6172d2552624 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:29:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e36cd namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: b80ff181-5923-4d46-bd76-93cfc7108201 kind: Event lastTimestamp: "2026-07-02T21:30:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:44Z" name: success-200-isvc-e36cd.18be964e4dd0ec11 namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 2bc686f4-c925-4910-a0dc-b6f8f17bce24 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e36cd namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: b80ff181-5923-4d46-bd76-93cfc7108201 kind: Event lastTimestamp: "2026-07-02T21:30:43Z" message: InferenceService [success-200-isvc-e36cd] is Ready metadata: creationTimestamp: "2026-07-02T21:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:43Z" name: success-200-isvc-e36cd.18be966091f0991f namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: ec59d943-65dd-4cce-9d16-ecf6ced02a0a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:40:24.834070Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz to ip-10-0-130-230.ec2.internal metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:40:24Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96e7cea8d8ab namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 0ed09fca-db94-4f2e-8cda-bcbf19de365c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:40:25Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96e7e82dc7a5 namespace: kserve-ci-e2e-test resourceVersion: "20178" uid: f9cdad31-3794-46ca-8df1-8afb50d7a653 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:25Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96e7e9886f32 namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: ddf7a9c6-a4cd-41cf-83f3-a172e5e20e4e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:25Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96e7ecb9a3e9 namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: c738c708-ac8d-44fb-b56a-635cb63a8b2b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:25Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96e7ed2f01ff namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 08836a69-6c1b-4ba8-a711-07451ddc4450 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:25Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96e7ed3a9f87 namespace: kserve-ci-e2e-test resourceVersion: "20185" uid: 501a61ac-c6c7-4f3c-b58f-82691151edb8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:25Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96e7f044455f namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: 4bbc1a11-e181-4e0a-a3e0-bcb0b58c3d9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:40:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:25Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96e7f0d609f1 namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 30afa922-9588-4dae-abcc-5158d97f73d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:41:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:01Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96e7fcad8b76 namespace: kserve-ci-e2e-test resourceVersion: "20640" uid: ee997b29-b356-494c-81c5-274ea82b9461 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96f829745761 namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: cd48da3b-8ad4-4f57-a4d0-91409b461205 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:41:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:35Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96f82974ebc3 namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 838fc032-2473-4add-b60c-9296e2e73efa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 4eedd1a9-82ce-49d5-8baa-9061c88c160f kind: Event lastTimestamp: "2026-07-02T21:41:36Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:41:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:36Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz.18be96f884c52221 namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: c6808f66-a33b-40dc-96cb-073dcc0909c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-230.ec2.internal source: component: kubelet host: ip-10-0-130-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ee70d-predictor-78b4c5c7b6 namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: 5acbfd18-7bf3-4f73-adf2-d0fc3a0aca38 kind: Event lastTimestamp: "2026-07-02T21:40:24Z" message: 'Created pod: success-200-isvc-ee70d-predictor-78b4c5c7b6-kv4jz' metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:24Z" name: success-200-isvc-ee70d-predictor-78b4c5c7b6.18be96e7ce40fa4e namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 13cb21ae-3144-417e-9fe8-69f96e9a26a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: 59987fed-a6c9-4c21-9c3b-994b828dd551 kind: Event lastTimestamp: "2026-07-02T21:40:24Z" message: Scaled up replica set success-200-isvc-ee70d-predictor-78b4c5c7b6 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:24Z" name: success-200-isvc-ee70d-predictor.18be96e7ca6c128a namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 62f51f19-d64b-4c05-8597-2f07f066e31e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:40:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 08cf1b12-b46f-4964-a868-1d36bd7572d1 kind: Event lastTimestamp: "2026-07-02T21:41:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:09Z" name: success-200-isvc-ee70d-predictor.18be96eb48eca225 namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: 1cd05c6d-d931-43ae-b887-e76941eed1c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:40:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 08cf1b12-b46f-4964-a868-1d36bd7572d1 kind: Event lastTimestamp: "2026-07-02T21:41:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:09Z" name: success-200-isvc-ee70d-predictor.18be96eb494f18e5 namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: b1a4ddfd-70a0-4155-993d-2679b572df5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 08cf1b12-b46f-4964-a868-1d36bd7572d1 kind: Event lastTimestamp: "2026-07-02T21:41:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:24Z" name: success-200-isvc-ee70d-predictor.18be96f5c556bc93 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: d8088ebd-1da8-429a-ade9-be3db6557f39 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ee70d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 08cf1b12-b46f-4964-a868-1d36bd7572d1 kind: Event lastTimestamp: "2026-07-02T21:41:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:24Z" name: success-200-isvc-ee70d-predictor.18be96f5c5b77e97 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 44e1b693-73d9-4e6b-b435-3d4d7f81f47e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ee70d namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: 22728a38-fa9c-4ef6-8ed8-4e997359d229 kind: Event lastTimestamp: "2026-07-02T21:41:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:11Z" name: success-200-isvc-ee70d.18be96e7ca72df63 namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: 2892e8e8-2c63-47fb-ae48-a504bec9b754 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ee70d namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: 22728a38-fa9c-4ef6-8ed8-4e997359d229 kind: Event lastTimestamp: "2026-07-02T21:41:11Z" message: InferenceService [success-200-isvc-ee70d] is Ready metadata: creationTimestamp: "2026-07-02T21:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:11Z" name: success-200-isvc-ee70d.18be96f2b7dd77d1 namespace: kserve-ci-e2e-test resourceVersion: "20736" uid: 9cef371b-c6eb-4b8f-a5aa-f4700c5eaf58 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:32:10.047002Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7615e-85ccdd4fcc-82s9x namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: db3c41f3-789c-42c6-a01e-7e67366cb754 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-7615e-85ccdd4fcc-82s9x to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:32:10Z" name: switch-graph-7615e-85ccdd4fcc-82s9x.18be96749b0d58fd namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: 89c52636-10d6-4b7b-bd26-332abd65477f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:10Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7615e-85ccdd4fcc-82s9x namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: db3c41f3-789c-42c6-a01e-7e67366cb754 kind: Event lastTimestamp: "2026-07-02T21:32:10Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:32:10Z" name: switch-graph-7615e-85ccdd4fcc-82s9x.18be9674b44f9eed namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: 4dfc2449-c41c-474a-9ff3-40b03fb06a59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7615e} kind: Pod name: switch-graph-7615e-85ccdd4fcc-82s9x namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: db3c41f3-789c-42c6-a01e-7e67366cb754 kind: Event lastTimestamp: "2026-07-02T21:32:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:10Z" name: switch-graph-7615e-85ccdd4fcc-82s9x.18be9674b589975d namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: 7017de0f-a6a3-4ea6-895b-7a5d471eecba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7615e} kind: Pod name: switch-graph-7615e-85ccdd4fcc-82s9x namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: db3c41f3-789c-42c6-a01e-7e67366cb754 kind: Event lastTimestamp: "2026-07-02T21:32:10Z" message: 'Created container: switch-graph-7615e' metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:10Z" name: switch-graph-7615e-85ccdd4fcc-82s9x.18be9674b84d28c1 namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: d67b362b-0612-4f35-908a-2f4417a00959 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7615e} kind: Pod name: switch-graph-7615e-85ccdd4fcc-82s9x namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: db3c41f3-789c-42c6-a01e-7e67366cb754 kind: Event lastTimestamp: "2026-07-02T21:32:10Z" message: Started container switch-graph-7615e metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:10Z" name: switch-graph-7615e-85ccdd4fcc-82s9x.18be9674b8ae96be namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: d89e8857-1f65-461f-944a-4bc922a5efab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7615e} kind: Pod name: switch-graph-7615e-85ccdd4fcc-82s9x namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: db3c41f3-789c-42c6-a01e-7e67366cb754 kind: Event lastTimestamp: "2026-07-02T21:40:24Z" message: Stopping container switch-graph-7615e metadata: creationTimestamp: "2026-07-02T21:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:24Z" name: switch-graph-7615e-85ccdd4fcc-82s9x.18be96e7c1fb35fb namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: e70f468c-9374-4b3e-b3ab-7afaedb9ae23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7615e} kind: Pod name: switch-graph-7615e-85ccdd4fcc-82s9x namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: db3c41f3-789c-42c6-a01e-7e67366cb754 kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: switch-graph-7615e-85ccdd4fcc-82s9x.18be96e83d810d13 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 4d7eaf0d-bb7f-4b19-ba46-75b6970eb908 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-7615e-85ccdd4fcc namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: b2326951-bebb-4df9-9748-9316090e3d60 kind: Event lastTimestamp: "2026-07-02T21:32:10Z" message: 'Created pod: switch-graph-7615e-85ccdd4fcc-82s9x' metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:10Z" name: switch-graph-7615e-85ccdd4fcc.18be96749a9c87f9 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: e72b8eb0-530d-4dce-8871-792a056f2256 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-7615e namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: 0ba7aa39-885f-488f-b7a7-a40a9a9a12d8 kind: Event lastTimestamp: "2026-07-02T21:32:10Z" message: Scaled up replica set switch-graph-7615e-85ccdd4fcc from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:10Z" name: switch-graph-7615e.18be967499ad0523 namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: 5012ac88-b234-4ca1-94e5-f89b4697d3e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-7615e namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: c807d418-6cb4-498c-b975-2c5360808030 kind: Event lastTimestamp: "2026-07-02T21:32:10Z" message: InferenceGraph [switch-graph-7615e] is Ready metadata: creationTimestamp: "2026-07-02T21:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:10Z" name: switch-graph-7615e.18be96749f5388ce namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: 8df0bd09-da98-4a75-916a-9bbdc51ef1b9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:32:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7615e namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: c5f31d7e-7aed-41a3-a94b-6259684a7eb6 kind: Event lastTimestamp: "2026-07-02T21:32:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:55Z" name: switch-graph-7615e.18be9678180b5c6c namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: 11e8742b-d33b-4514-8f35-c02acddc8a38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:32:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7615e namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: c5f31d7e-7aed-41a3-a94b-6259684a7eb6 kind: Event lastTimestamp: "2026-07-02T21:32:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:55Z" name: switch-graph-7615e.18be967818665577 namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: 0ea31144-647c-490f-920c-237bc2f79e31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:52:15.961996Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-76a5f-6849bc4b8-95t9c namespace: kserve-ci-e2e-test resourceVersion: "26785" uid: 9f04fadc-4d0a-4c57-a044-5433343392a0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-76a5f-6849bc4b8-95t9c to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:52:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:52:15Z" name: switch-graph-76a5f-6849bc4b8-95t9c.18be978d612fdc67 namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: 045db334-5e41-4785-87e7-1fafa04bb620 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:16Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-76a5f-6849bc4b8-95t9c namespace: kserve-ci-e2e-test resourceVersion: "26794" uid: 9f04fadc-4d0a-4c57-a044-5433343392a0 kind: Event lastTimestamp: "2026-07-02T21:52:16Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:52:16Z" name: switch-graph-76a5f-6849bc4b8-95t9c.18be978d7a468207 namespace: kserve-ci-e2e-test resourceVersion: "26808" uid: cb1b316c-7f9c-42f2-b15c-9ac6c4912c6f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-76a5f} kind: Pod name: switch-graph-76a5f-6849bc4b8-95t9c namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 9f04fadc-4d0a-4c57-a044-5433343392a0 kind: Event lastTimestamp: "2026-07-02T21:52:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:16Z" name: switch-graph-76a5f-6849bc4b8-95t9c.18be978d7b75a501 namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: 31e8d5a7-5f22-4a3c-aa4d-7f1f6de1c684 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-76a5f} kind: Pod name: switch-graph-76a5f-6849bc4b8-95t9c namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 9f04fadc-4d0a-4c57-a044-5433343392a0 kind: Event lastTimestamp: "2026-07-02T21:52:16Z" message: 'Created container: switch-graph-76a5f' metadata: creationTimestamp: "2026-07-02T21:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:16Z" name: switch-graph-76a5f-6849bc4b8-95t9c.18be978d7e9379d0 namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 81f37148-bb34-4536-b872-fc1c732792e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-76a5f} kind: Pod name: switch-graph-76a5f-6849bc4b8-95t9c namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 9f04fadc-4d0a-4c57-a044-5433343392a0 kind: Event lastTimestamp: "2026-07-02T21:52:16Z" message: Started container switch-graph-76a5f metadata: creationTimestamp: "2026-07-02T21:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:52:16Z" name: switch-graph-76a5f-6849bc4b8-95t9c.18be978d7f0205c8 namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: 61e9dbf5-c702-4b0a-a115-e5f51d8bcc2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T22:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-76a5f} kind: Pod name: switch-graph-76a5f-6849bc4b8-95t9c namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 9f04fadc-4d0a-4c57-a044-5433343392a0 kind: Event lastTimestamp: "2026-07-02T22:08:35Z" message: Stopping container switch-graph-76a5f metadata: creationTimestamp: "2026-07-02T22:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:08:35Z" name: switch-graph-76a5f-6849bc4b8-95t9c.18be9871627523ee namespace: kserve-ci-e2e-test resourceVersion: "33683" uid: b037022d-05e4-4136-a7f2-14acfad27a10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T22:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-76a5f} kind: Pod name: switch-graph-76a5f-6849bc4b8-95t9c namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 9f04fadc-4d0a-4c57-a044-5433343392a0 kind: Event lastTimestamp: "2026-07-02T22:08:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T22:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T22:08:58Z" name: switch-graph-76a5f-6849bc4b8-95t9c.18be98720f2cdbc6 namespace: kserve-ci-e2e-test resourceVersion: "33922" uid: b8958e8f-a985-4563-9679-1ebc69aef087 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-76a5f-6849bc4b8 namespace: kserve-ci-e2e-test resourceVersion: "26782" uid: 376c87e0-90b8-4069-aa24-c877eb496517 kind: Event lastTimestamp: "2026-07-02T21:52:15Z" message: 'Created pod: switch-graph-76a5f-6849bc4b8-95t9c' metadata: creationTimestamp: "2026-07-02T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:15Z" name: switch-graph-76a5f-6849bc4b8.18be978d60866676 namespace: kserve-ci-e2e-test resourceVersion: "26786" uid: a71cb74e-9fdb-4591-a732-b7aea19af9c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-76a5f namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: c467943e-d5de-41c5-bf40-4a51239a3677 kind: Event lastTimestamp: "2026-07-02T21:52:15Z" message: Scaled up replica set switch-graph-76a5f-6849bc4b8 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:15Z" name: switch-graph-76a5f.18be978d5fd4bee8 namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: c1a5a660-23d4-460c-8f4c-f794a53c8139 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-76a5f namespace: kserve-ci-e2e-test resourceVersion: "26799" uid: eff85bea-f289-4745-ab7a-a3f3f2f13f46 kind: Event lastTimestamp: "2026-07-02T21:52:16Z" message: InferenceGraph [switch-graph-76a5f] is Ready metadata: creationTimestamp: "2026-07-02T21:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:52:16Z" name: switch-graph-76a5f.18be978d650477ba namespace: kserve-ci-e2e-test resourceVersion: "26801" uid: e83c68d4-7374-4db3-b036-8496ba331b71 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-76a5f namespace: kserve-ci-e2e-test resourceVersion: "26781" uid: f50dcdef-e00c-408c-8c03-9c8faaede1e9 kind: Event lastTimestamp: "2026-07-02T21:52:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:52:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:30Z" name: switch-graph-76a5f.18be9790de37bcec namespace: kserve-ci-e2e-test resourceVersion: "26932" uid: d8742c64-9a97-4633-bdcc-a77e35665231 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-76a5f namespace: kserve-ci-e2e-test resourceVersion: "26781" uid: f50dcdef-e00c-408c-8c03-9c8faaede1e9 kind: Event lastTimestamp: "2026-07-02T21:52:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:52:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:30Z" name: switch-graph-76a5f.18be9790de99aecf namespace: kserve-ci-e2e-test resourceVersion: "26934" uid: 848672cd-495f-4f5d-9ce2-0947f4389f0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-76a5f namespace: kserve-ci-e2e-test resourceVersion: "26933" uid: f50dcdef-e00c-408c-8c03-9c8faaede1e9 kind: Event lastTimestamp: "2026-07-02T21:52:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:45Z" name: switch-graph-76a5f.18be97945d4b431d namespace: kserve-ci-e2e-test resourceVersion: "27043" uid: bc20f170-cf51-4d49-9020-e0b02256742b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:52:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-76a5f namespace: kserve-ci-e2e-test resourceVersion: "26933" uid: f50dcdef-e00c-408c-8c03-9c8faaede1e9 kind: Event lastTimestamp: "2026-07-02T21:52:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:52:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:52:45Z" name: switch-graph-76a5f.18be97945d934f00 namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: 608ab08a-717a-4478-bc02-80c65ae3773a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:30:55.573013Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e36cd-68f8c7995c-glw64 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: af504310-588c-43b8-912b-2c56a6dd7992 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e36cd-68f8c7995c-glw64 to ip-10-0-131-12.ec2.internal metadata: creationTimestamp: "2026-07-02T21:30:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:30:55Z" name: switch-graph-e36cd-68f8c7995c-glw64.18be9663440e7bf0 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: eddb7cc9-2b5d-40c4-8372-4e35f033d540 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7dcc8b86f5-nwgkq source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:30:55Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e36cd-68f8c7995c-glw64 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: af504310-588c-43b8-912b-2c56a6dd7992 kind: Event lastTimestamp: "2026-07-02T21:30:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-e36cd-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:56Z" name: switch-graph-e36cd-68f8c7995c-glw64.18be96634f2c5673 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: 04c89116-106a-4b70-9a10-13c094ae55da reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:57Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e36cd-68f8c7995c-glw64 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: af504310-588c-43b8-912b-2c56a6dd7992 kind: Event lastTimestamp: "2026-07-02T21:30:57Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:30:57Z" name: switch-graph-e36cd-68f8c7995c-glw64.18be9663b6a92030 namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: cda2107d-9bd1-4143-97b6-0f483153398e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e36cd} kind: Pod name: switch-graph-e36cd-68f8c7995c-glw64 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: af504310-588c-43b8-912b-2c56a6dd7992 kind: Event lastTimestamp: "2026-07-02T21:30:57Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" metadata: creationTimestamp: "2026-07-02T21:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:57Z" name: switch-graph-e36cd-68f8c7995c-glw64.18be9663b7d903ea namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: cf7375c8-3f3b-4825-af35-837013d1b555 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e36cd} kind: Pod name: switch-graph-e36cd-68f8c7995c-glw64 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: af504310-588c-43b8-912b-2c56a6dd7992 kind: Event lastTimestamp: "2026-07-02T21:31:00Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" in 2.49s (2.49s including waiting). Image size: 218072672 bytes.' metadata: creationTimestamp: "2026-07-02T21:31:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:00Z" name: switch-graph-e36cd-68f8c7995c-glw64.18be96644c4ca933 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 8b2692dd-3c53-4a7e-862c-c25915904ab7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e36cd} kind: Pod name: switch-graph-e36cd-68f8c7995c-glw64 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: af504310-588c-43b8-912b-2c56a6dd7992 kind: Event lastTimestamp: "2026-07-02T21:31:00Z" message: 'Created container: switch-graph-e36cd' metadata: creationTimestamp: "2026-07-02T21:31:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:00Z" name: switch-graph-e36cd-68f8c7995c-glw64.18be96644fe3ae11 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: cfde6346-46b7-4700-b0cf-520d021996e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e36cd} kind: Pod name: switch-graph-e36cd-68f8c7995c-glw64 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: af504310-588c-43b8-912b-2c56a6dd7992 kind: Event lastTimestamp: "2026-07-02T21:31:00Z" message: Started container switch-graph-e36cd metadata: creationTimestamp: "2026-07-02T21:31:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:00Z" name: switch-graph-e36cd-68f8c7995c-glw64.18be966450550010 namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 11081b29-46d6-4d2e-8b54-4e63c8f207df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e36cd} kind: Pod name: switch-graph-e36cd-68f8c7995c-glw64 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: af504310-588c-43b8-912b-2c56a6dd7992 kind: Event lastTimestamp: "2026-07-02T21:31:09Z" message: Stopping container switch-graph-e36cd metadata: creationTimestamp: "2026-07-02T21:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:09Z" name: switch-graph-e36cd-68f8c7995c-glw64.18be966695d888e9 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 9fb26a1d-6307-4eac-ad50-411d094342bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e36cd} kind: Pod name: switch-graph-e36cd-68f8c7995c-glw64 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: af504310-588c-43b8-912b-2c56a6dd7992 kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:36Z" name: switch-graph-e36cd-68f8c7995c-glw64.18be9666f5408563 namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: 4daf1e63-23ce-44e0-a192-eeb3345137eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-12.ec2.internal source: component: kubelet host: ip-10-0-131-12.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e36cd-68f8c7995c namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: b464bbd7-2e6e-47f7-9282-fc381411cfa2 kind: Event lastTimestamp: "2026-07-02T21:30:55Z" message: 'Created pod: switch-graph-e36cd-68f8c7995c-glw64' metadata: creationTimestamp: "2026-07-02T21:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:55Z" name: switch-graph-e36cd-68f8c7995c.18be966343b651a1 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 207a8107-d783-435f-9883-9e6818f09571 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e36cd namespace: kserve-ci-e2e-test resourceVersion: "15533" uid: 48365e14-3dab-467d-8482-e9c6b744e35d kind: Event lastTimestamp: "2026-07-02T21:30:55Z" message: Scaled up replica set switch-graph-e36cd-68f8c7995c from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:55Z" name: switch-graph-e36cd.18be96634290c2f6 namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: 95778a82-bbf1-4d05-a80e-225904c7a241 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e36cd namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: 286bbf02-f6d7-4f9c-8fc7-ca3ba52a2364 kind: Event lastTimestamp: "2026-07-02T21:30:55Z" message: InferenceGraph [switch-graph-e36cd] is Ready metadata: creationTimestamp: "2026-07-02T21:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:55Z" name: switch-graph-e36cd.18be9663484ac56b namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: 8d152fda-f805-40de-a96e-c1d7fcbd7378 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33953"