--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:53:43.529428Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ac602-7c945b5b88-wwhgf namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: b3c11182-aa87-45d2-af1e-3d53777fccea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-ac602-7c945b5b88-wwhgf to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:53:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:53:43Z" name: ensemble-graph-ac602-7c945b5b88-wwhgf.18a73327ad8db734 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 6b894896-0fc3-4aeb-8e89-295a2183306d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:43Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ac602-7c945b5b88-wwhgf namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: b3c11182-aa87-45d2-af1e-3d53777fccea kind: Event lastTimestamp: "2026-04-17T16:53:43Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:53:43Z" name: ensemble-graph-ac602-7c945b5b88-wwhgf.18a73327c6917e94 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 49b46007-3aea-4807-8b98-93e73e54701e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac602} kind: Pod name: ensemble-graph-ac602-7c945b5b88-wwhgf namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: b3c11182-aa87-45d2-af1e-3d53777fccea kind: Event lastTimestamp: "2026-04-17T16:53:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:43Z" name: ensemble-graph-ac602-7c945b5b88-wwhgf.18a73327c7d4861b namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 1bb3012f-29df-4de3-812b-bc148d7d3540 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac602} kind: Pod name: ensemble-graph-ac602-7c945b5b88-wwhgf namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: b3c11182-aa87-45d2-af1e-3d53777fccea kind: Event lastTimestamp: "2026-04-17T16:53:44Z" message: 'Created container: ensemble-graph-ac602' metadata: creationTimestamp: "2026-04-17T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:44Z" name: ensemble-graph-ac602-7c945b5b88-wwhgf.18a73327cbd74078 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0e5a0dbf-0d81-4084-b58c-b7f81cfca04b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac602} kind: Pod name: ensemble-graph-ac602-7c945b5b88-wwhgf namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: b3c11182-aa87-45d2-af1e-3d53777fccea kind: Event lastTimestamp: "2026-04-17T16:53:44Z" message: Started container ensemble-graph-ac602 metadata: creationTimestamp: "2026-04-17T16:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:44Z" name: ensemble-graph-ac602-7c945b5b88-wwhgf.18a73327cc47bae4 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 64297bc7-9422-45e1-823f-cc05af24f442 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac602} kind: Pod name: ensemble-graph-ac602-7c945b5b88-wwhgf namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: b3c11182-aa87-45d2-af1e-3d53777fccea kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: Stopping container ensemble-graph-ac602 metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:53Z" name: ensemble-graph-ac602-7c945b5b88-wwhgf.18a7332a04f47079 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 0704c8ff-e781-49b5-a08f-cd3cc9911d55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac602} kind: Pod name: ensemble-graph-ac602-7c945b5b88-wwhgf namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: b3c11182-aa87-45d2-af1e-3d53777fccea kind: Event lastTimestamp: "2026-04-17T16:54:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:20Z" name: ensemble-graph-ac602-7c945b5b88-wwhgf.18a7332a5e2bc7c3 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 466df468-b37d-41a4-9e0e-61139ecbc8da reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-ac602-7c945b5b88 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 555de499-fc78-4d54-90a9-6dad27d1b071 kind: Event lastTimestamp: "2026-04-17T16:53:43Z" message: 'Created pod: ensemble-graph-ac602-7c945b5b88-wwhgf' metadata: creationTimestamp: "2026-04-17T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:43Z" name: ensemble-graph-ac602-7c945b5b88.18a73327ad28989f namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 4a218013-0a2c-454e-8413-d36f177ef181 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-ac602 namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: 50835bf2-6e68-4bfa-b924-12a3a7209e38 kind: Event lastTimestamp: "2026-04-17T16:53:43Z" message: Scaled up replica set ensemble-graph-ac602-7c945b5b88 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:43Z" name: ensemble-graph-ac602.18a73327ac26884f namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 78c7a8f3-93fd-4250-b2cd-be4c49d5a2d1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ac602 namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 0a599d87-89f5-41e3-a9fa-f62db68be16a kind: Event lastTimestamp: "2026-04-17T16:53:43Z" message: InferenceGraph [ensemble-graph-ac602] is Ready metadata: creationTimestamp: "2026-04-17T16:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:43Z" name: ensemble-graph-ac602.18a73327b1567577 namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: a53a3199-f587-40b1-ab9f-7941b470d061 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:55:03.814566Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c68dd-c6797d99d-xsj96 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 88b45c30-55a8-411a-9555-1cfce612409c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c68dd-c6797d99d-xsj96 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:55:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:55:03Z" name: ensemble-graph-c68dd-c6797d99d-xsj96.18a7333a5eeba3f7 namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: f3c5a901-d61e-4606-94f4-4b189afd69da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:04Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c68dd-c6797d99d-xsj96 namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: 88b45c30-55a8-411a-9555-1cfce612409c kind: Event lastTimestamp: "2026-04-17T16:55:04Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:55:04Z" name: ensemble-graph-c68dd-c6797d99d-xsj96.18a7333a780f789d namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: d006fd2a-b903-48dc-9010-cf84357dc8fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c68dd} kind: Pod name: ensemble-graph-c68dd-c6797d99d-xsj96 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 88b45c30-55a8-411a-9555-1cfce612409c kind: Event lastTimestamp: "2026-04-17T16:55:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:55:04Z" name: ensemble-graph-c68dd-c6797d99d-xsj96.18a7333a7a84f5a7 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 039ef7f6-3ec1-4d3c-aff2-27fd17cc290d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c68dd} kind: Pod name: ensemble-graph-c68dd-c6797d99d-xsj96 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 88b45c30-55a8-411a-9555-1cfce612409c kind: Event lastTimestamp: "2026-04-17T16:55:04Z" message: 'Created container: ensemble-graph-c68dd' metadata: creationTimestamp: "2026-04-17T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:55:04Z" name: ensemble-graph-c68dd-c6797d99d-xsj96.18a7333a7d8078ba namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: abdfbf5c-0d1b-4084-bb91-9adf64e7a724 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c68dd} kind: Pod name: ensemble-graph-c68dd-c6797d99d-xsj96 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 88b45c30-55a8-411a-9555-1cfce612409c kind: Event lastTimestamp: "2026-04-17T16:55:04Z" message: Started container ensemble-graph-c68dd metadata: creationTimestamp: "2026-04-17T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:55:04Z" name: ensemble-graph-c68dd-c6797d99d-xsj96.18a7333a7deff8bf namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 170de448-2249-4659-be45-7f11d4961be7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c68dd} kind: Pod name: ensemble-graph-c68dd-c6797d99d-xsj96 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 88b45c30-55a8-411a-9555-1cfce612409c kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: Stopping container ensemble-graph-c68dd metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:18Z" name: ensemble-graph-c68dd-c6797d99d-xsj96.18a733ad94aaf251 namespace: kserve-ci-e2e-test resourceVersion: "24936" uid: 1ae00bb6-b280-45bf-a18a-9583073a3658 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c68dd} kind: Pod name: ensemble-graph-c68dd-c6797d99d-xsj96 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 88b45c30-55a8-411a-9555-1cfce612409c kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: ensemble-graph-c68dd-c6797d99d-xsj96.18a733adfb83a905 namespace: kserve-ci-e2e-test resourceVersion: "25565" uid: 3dbfe527-b856-49ef-a832-726652764444 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c68dd-c6797d99d namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 32b03245-484f-4ee3-b662-b9b8baf7d014 kind: Event lastTimestamp: "2026-04-17T16:55:03Z" message: 'Created pod: ensemble-graph-c68dd-c6797d99d-xsj96' metadata: creationTimestamp: "2026-04-17T16:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:03Z" name: ensemble-graph-c68dd-c6797d99d.18a7333a5e78b1d8 namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: a040009c-941a-4286-87bc-b3f79cf0084a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c68dd namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 106c32a2-8b80-4252-b590-5d7b315c31e4 kind: Event lastTimestamp: "2026-04-17T16:55:03Z" message: Scaled up replica set ensemble-graph-c68dd-c6797d99d from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:03Z" name: ensemble-graph-c68dd.18a7333a5da99b50 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 61a1bca6-0289-48d1-b6c9-a73a7aa24a70 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c68dd namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: 9d699c09-cb38-4d29-9412-1ab0dd0163b4 kind: Event lastTimestamp: "2026-04-17T16:55:03Z" message: InferenceGraph [ensemble-graph-c68dd] is Ready metadata: creationTimestamp: "2026-04-17T16:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:55:03Z" name: ensemble-graph-c68dd.18a7333a66eb20fc namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 3dd9e5f3-cdb7-453a-9bbf-9e66dc4f3bd5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c68dd namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: 52a0e5c3-87c9-4e29-910c-4e6c16599317 kind: Event lastTimestamp: "2026-04-17T16:55:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:18Z" name: ensemble-graph-c68dd.18a7333ddc38daff namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: 5a5ecde3-d963-4786-a5dc-634b2dc9a183 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c68dd namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: 52a0e5c3-87c9-4e29-910c-4e6c16599317 kind: Event lastTimestamp: "2026-04-17T16:55:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:18Z" name: ensemble-graph-c68dd.18a7333ddc9500f0 namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 1cfc26e1-e64c-4303-adbb-4b4893f8150d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c68dd namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 52a0e5c3-87c9-4e29-910c-4e6c16599317 kind: Event lastTimestamp: "2026-04-17T16:55:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:33Z" name: ensemble-graph-c68dd.18a733415b39f0bf namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 75cddc08-a137-48d2-80f9-a075d09c3f56 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c68dd namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 52a0e5c3-87c9-4e29-910c-4e6c16599317 kind: Event lastTimestamp: "2026-04-17T16:55:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:33Z" name: ensemble-graph-c68dd.18a733415b9dbd45 namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: d6007e3a-d08f-41fd-a3a0-9f2c8f6e6727 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:03:18.913872Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25011" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-41c4d-predictor-ffc88656f-4gvqb to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:03:18Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733ada52399f2 namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: a66758dc-4b21-45ed-abb9-37286a4c938d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:03:19Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733adbe906fa6 namespace: kserve-ci-e2e-test resourceVersion: "25033" uid: c49c9e30-c6c1-4716-96bc-941db31b7c91 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733adbfe66ca4 namespace: kserve-ci-e2e-test resourceVersion: "25036" uid: 20dcf28d-6153-47b6-b0a4-bed9331ba5eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733adc299982f namespace: kserve-ci-e2e-test resourceVersion: "25041" uid: e98cc04e-9736-4420-9daa-6840db924018 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733adc30df120 namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: af1ce2c3-4b98-4542-ad57-78a7b39957ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733adc31888f3 namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 3c1fd0c0-2cf0-487c-80de-67d1c5dfe1c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733adc6167d02 namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: 2b549c1d-8486-4c50-81e2-2b3e81bdd035 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733adc6859ba5 namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: b374a38d-f40f-4f04-aad5-dca1d9100c6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:03:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:56Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733ae13e057c4 namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: bb5d6481-b7da-4a07-9172-7727339c00d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:29Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733bdfb9f7faf namespace: kserve-ci-e2e-test resourceVersion: "26183" uid: 79099700-416a-4217-8457-8491c990cfd7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:29Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733bdfba0b4cd namespace: kserve-ci-e2e-test resourceVersion: "26186" uid: 97c9643d-293f-4553-b4aa-7d3ac9f476b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: d1ecf353-5c1a-48eb-8bd5-66b1a725f4bb kind: Event lastTimestamp: "2026-04-17T17:04:31Z" message: 'Readiness probe failed: Get "https://10.132.0.44:8643/healthz": dial tcp 10.132.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:31Z" name: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb.18a733be9bf8454a namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: c721c634-0912-4c2e-b0d2-7ff5466ffc8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-41c4d-predictor-ffc88656f namespace: kserve-ci-e2e-test resourceVersion: "25000" uid: 9735e415-76e2-42cc-ae82-c25f287b873b kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: 'Created pod: error-404-isvc-41c4d-predictor-ffc88656f-4gvqb' metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:18Z" name: error-404-isvc-41c4d-predictor-ffc88656f.18a733ada4bf304c namespace: kserve-ci-e2e-test resourceVersion: "25013" uid: faffdc4e-11a4-457a-8420-bf7490fc2b17 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "24997" uid: 0b8c811a-567f-42be-9c94-a3a0d97d5004 kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: Scaled up replica set error-404-isvc-41c4d-predictor-ffc88656f from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:18Z" name: error-404-isvc-41c4d-predictor.18a733ada28665b9 namespace: kserve-ci-e2e-test resourceVersion: "25003" uid: 6cbe7521-da19-4fc2-b833-171f819dadf9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25002" uid: d00fb2a0-d1c3-423f-b143-837946cbbe70 kind: Event lastTimestamp: "2026-04-17T17:03:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:48Z" name: error-404-isvc-41c4d-predictor.18a733b121434028 namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: d2ea11bd-beac-42fa-8ee3-be15beb52992 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25002" uid: d00fb2a0-d1c3-423f-b143-837946cbbe70 kind: Event lastTimestamp: "2026-04-17T17:03:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:48Z" name: error-404-isvc-41c4d-predictor.18a733b121b1e13b namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: b57183c0-bbdf-49e5-b694-70d6852037ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:04:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: d00fb2a0-d1c3-423f-b143-837946cbbe70 kind: Event lastTimestamp: "2026-04-17T17:04:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:18Z" name: error-404-isvc-41c4d-predictor.18a733b81f56a525 namespace: kserve-ci-e2e-test resourceVersion: "26022" uid: 5151d399-5828-4157-b151-8456db210eea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:04:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: d00fb2a0-d1c3-423f-b143-837946cbbe70 kind: Event lastTimestamp: "2026-04-17T17:04:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:18Z" name: error-404-isvc-41c4d-predictor.18a733b81f9c802f namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: 981c7b00-6bf6-45cb-b516-924fbc91b788 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-41c4d namespace: kserve-ci-e2e-test resourceVersion: "24972" uid: db3c6926-d735-418b-b02b-8fe2be5fc3fa kind: Event lastTimestamp: "2026-04-17T17:04:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:06Z" name: error-404-isvc-41c4d.18a733ada2a87692 namespace: kserve-ci-e2e-test resourceVersion: "25897" uid: 65d0993e-a11a-4a5a-a1b6-64bc033d808d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-41c4d namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: db3c6926-d735-418b-b02b-8fe2be5fc3fa kind: Event lastTimestamp: "2026-04-17T17:04:06Z" message: InferenceService [error-404-isvc-41c4d] is Ready metadata: creationTimestamp: "2026-04-17T17:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:06Z" name: error-404-isvc-41c4d.18a733b8cf02345e namespace: kserve-ci-e2e-test resourceVersion: "25893" uid: 69e75e55-87a0-4263-9d1d-b439254537fb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:03:44.457409Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25514" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6941b-predictor-79dfb7969c-rn424 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:03:44Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b397a708fd namespace: kserve-ci-e2e-test resourceVersion: "25518" uid: 9982bc20-d1b5-4323-9bc4-8cad95d91aae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6941b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:44Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b3a689c388 namespace: kserve-ci-e2e-test resourceVersion: "25530" uid: 295a539c-c259-4e9f-acdd-fddfb1c159a8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25524" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:03:45Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b3d573aed3 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: a7a49976-2497-4d36-8405-fc310450f07d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b3d73dfcc7 namespace: kserve-ci-e2e-test resourceVersion: "25576" uid: cbba1a5f-2d1b-4c7a-8455-c7a9af5186d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b3d964b4e7 namespace: kserve-ci-e2e-test resourceVersion: "25579" uid: d15087b6-a389-46ef-ae0d-d1e9c179a980 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b3d9ce3950 namespace: kserve-ci-e2e-test resourceVersion: "25580" uid: e8afaf5d-8de6-4f96-83ff-9a048cc21548 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b3d9d989bc namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 1dc0ab56-5603-4227-8dd3-052844682acf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b3dda33bf8 namespace: kserve-ci-e2e-test resourceVersion: "25586" uid: 8a4ee201-0b7b-4c0f-b098-bea3bb0b0f44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b3de170299 namespace: kserve-ci-e2e-test resourceVersion: "25587" uid: 1faa57dd-b6a7-4022-aceb-9b20e64299df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:04:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:31Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a733b3eb3a7651 namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: c579fc45-433b-4a01-ae83-9c8d9afca36e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:21:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:21:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:21:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:21:14Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a734a7f70a644c namespace: kserve-ci-e2e-test resourceVersion: "33493" uid: 58f95d8f-ad31-40c8-8bad-7008ee774d8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:21:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:21:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:21:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:21:14Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a734a7f70aceb2 namespace: kserve-ci-e2e-test resourceVersion: "33494" uid: 0f440c35-c02f-40bd-87be-f854f675d74e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:21:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6941b-predictor-79dfb7969c-rn424 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: b3e64c5f-1499-4d3c-b9b1-6a09295446b4 kind: Event lastTimestamp: "2026-04-17T17:21:16Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:21:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:21:16Z" name: error-404-isvc-6941b-predictor-79dfb7969c-rn424.18a734a89fe0142e namespace: kserve-ci-e2e-test resourceVersion: "33546" uid: bbdf1657-06e4-40b9-9106-7c8b48a61a81 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6941b-predictor-79dfb7969c namespace: kserve-ci-e2e-test resourceVersion: "25507" uid: 8afe85b5-60fb-4117-b999-dbd0cabe38ee kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: 'Created pod: error-404-isvc-6941b-predictor-79dfb7969c-rn424' metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:44Z" name: error-404-isvc-6941b-predictor-79dfb7969c.18a733b3971ae63f namespace: kserve-ci-e2e-test resourceVersion: "25515" uid: 99d8d1e8-f060-4777-9f9c-a4613d62be6d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25505" uid: e105b8b3-1eae-4689-945c-fbc73849469a kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: Scaled up replica set error-404-isvc-6941b-predictor-79dfb7969c from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:44Z" name: error-404-isvc-6941b-predictor.18a733b395959e5d namespace: kserve-ci-e2e-test resourceVersion: "25509" uid: 097d1174-32b7-4a05-83fb-642e829b26eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25506" uid: 0e3fe9be-0606-488d-a310-98668b9a080b kind: Event lastTimestamp: "2026-04-17T17:04:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:14Z" name: error-404-isvc-6941b-predictor.18a733b71414415f namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: 2416944f-3e7d-4ce3-8dfc-6eeed6de62aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25506" uid: 0e3fe9be-0606-488d-a310-98668b9a080b kind: Event lastTimestamp: "2026-04-17T17:04:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:14Z" name: error-404-isvc-6941b-predictor.18a733b71458482f namespace: kserve-ci-e2e-test resourceVersion: "25957" uid: e3f34282-0201-4943-8e22-b8b71c62a91c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25822" uid: 0e3fe9be-0606-488d-a310-98668b9a080b kind: Event lastTimestamp: "2026-04-17T17:04:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:59Z" name: error-404-isvc-6941b-predictor.18a733be11c5001d namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: b3e7fc1e-dc12-4e53-88d3-73e1d0770880 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25822" uid: 0e3fe9be-0606-488d-a310-98668b9a080b kind: Event lastTimestamp: "2026-04-17T17:04:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:59Z" name: error-404-isvc-6941b-predictor.18a733be120ec9a0 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: 8ef89e44-78f6-420e-a11e-b9e6d904a272 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 56f10863-9199-4501-86bc-465ff528d504 kind: Event lastTimestamp: "2026-04-17T17:04:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:41Z" name: error-404-isvc-6941b.18a733b3957400d9 namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: ce0c5e6b-6501-48e4-9715-3c1a42b4a800 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 56f10863-9199-4501-86bc-465ff528d504 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-6941b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6941b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:03:44Z" name: error-404-isvc-6941b.18a733b395dba2ea namespace: kserve-ci-e2e-test resourceVersion: "25511" uid: 4e44513e-4308-4921-b8a1-fecba559856b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 56f10863-9199-4501-86bc-465ff528d504 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6941b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:03:44Z" name: error-404-isvc-6941b.18a733b395dbeac5 namespace: kserve-ci-e2e-test resourceVersion: "25512" uid: cef272ce-81a5-442f-8c24-ba57a2d6ee16 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 56f10863-9199-4501-86bc-465ff528d504 kind: Event lastTimestamp: "2026-04-17T17:04:41Z" message: InferenceService [error-404-isvc-6941b] is Ready metadata: creationTimestamp: "2026-04-17T17:04:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:41Z" name: error-404-isvc-6941b.18a733c0f9c2c103 namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: 693b7ab0-d5a3-4f0f-acea-aa763a4689a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:04:29.179406Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:04:29Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733be014a7bf3 namespace: kserve-ci-e2e-test resourceVersion: "26206" uid: 79675340-a7e6-406e-a169-999800255f8f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7afa6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:29Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733be0d4ed927 namespace: kserve-ci-e2e-test resourceVersion: "26221" uid: e12cea3f-6258-4c26-ae5d-41bd0c4507e0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Add eth0 [10.132.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:04:30Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733be3e860961 namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: 020e7ca1-4712-4cc9-82cb-0b8e9a7b59a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733be40041a66 namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: 247d08b7-3008-4bbe-bf25-8a5c9c5ad436 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733be432ba276 namespace: kserve-ci-e2e-test resourceVersion: "26253" uid: fab2fd68-5858-41a0-ade7-8581d29078db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733be4398d449 namespace: kserve-ci-e2e-test resourceVersion: "26254" uid: bdb0c824-fd85-40b1-8c70-314c965dc545 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733be43a4bf6e namespace: kserve-ci-e2e-test resourceVersion: "26255" uid: 92ebd962-d9be-4f92-85ae-0594be815e63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733be46cb6b88 namespace: kserve-ci-e2e-test resourceVersion: "26256" uid: 8b1566e5-4b42-4bbc-9802-2089d5ad6f31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733be474feddf namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: ad9f35d9-6bb0-49f9-bc09-9d5d9e642aa9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:04:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:05:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:05:08Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a733beaa7e4e2d namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 01d409cc-6e03-4def-b4b9-277e7cd9406f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:13:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:13:53Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a7344181c254ba namespace: kserve-ci-e2e-test resourceVersion: "30422" uid: 614f898f-6dc7-4bef-94e4-b817575086ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: 16af9bfe-cb61-44e9-bb37-385d3eefa146 kind: Event lastTimestamp: "2026-04-17T17:13:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:13:53Z" name: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82.18a7344181c2ce31 namespace: kserve-ci-e2e-test resourceVersion: "30423" uid: f8a713ac-abf8-4a79-a232-e4ad6fade691 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7afa6-predictor-5f65ffc855 namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: 253802e0-e209-45ae-bcea-3f1bf8be0d1e kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: 'Created pod: error-404-isvc-7afa6-predictor-5f65ffc855-lhn82' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:29Z" name: error-404-isvc-7afa6-predictor-5f65ffc855.18a733be009e8a73 namespace: kserve-ci-e2e-test resourceVersion: "26203" uid: 516fb26e-1bf0-4d7e-8aad-6ad33fcd9295 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: a6118c77-ecb2-4635-a930-cc5364cbe5bc kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: Scaled up replica set error-404-isvc-7afa6-predictor-5f65ffc855 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:29Z" name: error-404-isvc-7afa6-predictor.18a733bdfed71d67 namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: 5a65e559-2873-4de2-8d6d-b4cf12368ff7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:04:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: 078b692b-2ab8-46ee-b876-70779599fce4 kind: Event lastTimestamp: "2026-04-17T17:04:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:59Z" name: error-404-isvc-7afa6-predictor.18a733c17d4bf348 namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: 8d08ce8b-f780-4c82-b11b-082015068442 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:04:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: 078b692b-2ab8-46ee-b876-70779599fce4 kind: Event lastTimestamp: "2026-04-17T17:04:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:59Z" name: error-404-isvc-7afa6-predictor.18a733c17dab8a2e namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 7cea02ad-45f9-4f57-968f-f6e0e762950b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26399" uid: 078b692b-2ab8-46ee-b876-70779599fce4 kind: Event lastTimestamp: "2026-04-17T17:05:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:44Z" name: error-404-isvc-7afa6-predictor.18a733c87b1551f5 namespace: kserve-ci-e2e-test resourceVersion: "26958" uid: b4359edd-3804-438b-85ac-9eeba9261016 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26399" uid: 078b692b-2ab8-46ee-b876-70779599fce4 kind: Event lastTimestamp: "2026-04-17T17:05:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:44Z" name: error-404-isvc-7afa6-predictor.18a733c87b77463c namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 3ae06308-d7ff-4b9d-b2a6-2fb9d039530a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: 6e104c98-a569-416b-b7dc-c898841e35ff kind: Event lastTimestamp: "2026-04-17T17:05:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:05:18Z" name: error-404-isvc-7afa6.18a733bdfea5e0b6 namespace: kserve-ci-e2e-test resourceVersion: "26722" uid: cf22cb69-9ecc-40c9-91e9-217ae2613d6b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: 6e104c98-a569-416b-b7dc-c898841e35ff kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-7afa6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7afa6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:29Z" name: error-404-isvc-7afa6.18a733bdff29e99d namespace: kserve-ci-e2e-test resourceVersion: "26199" uid: e7d6a370-dfbb-49fa-bd1a-74fa0bf21ab8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: 6e104c98-a569-416b-b7dc-c898841e35ff kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7afa6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:29Z" name: error-404-isvc-7afa6.18a733bdff2a837b namespace: kserve-ci-e2e-test resourceVersion: "26200" uid: 146047a4-e457-44f3-92b2-6a5e9bb79e1e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 6e104c98-a569-416b-b7dc-c898841e35ff kind: Event lastTimestamp: "2026-04-17T17:05:18Z" message: InferenceService [error-404-isvc-7afa6] is Ready metadata: creationTimestamp: "2026-04-17T17:05:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:05:18Z" name: error-404-isvc-7afa6.18a733c965823b2f namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: eb95cc44-951f-47f9-8a8b-b73463b67a53 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:43:54.492099Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:43:54Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329e883167ff namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 1b843f3f-1db5-4d73-ab05-7289b8e3c57a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8b822-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329e93b37914 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 32d42c76-44e5-4ac9-83e1-92f90cefb3f8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:43:55Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329ec57037a2 namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 519db210-aa07-4dfd-a00b-fc1e530d8b64 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329ec6d4a738 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 2bc0c637-469b-4ba9-bdd6-14dd032c36a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329ec99d3f02 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: 261a713c-90fc-4aee-bd28-287fe3902525 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329eca0bc63c namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 57657aad-831c-4f90-9751-23ee82b16757 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329eca17609c namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 7e8b1e17-f0ce-420e-8329-dadd04744599 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329ecce1fb0a namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: c9bb52d4-83d9-4d86-999b-3eb063f422b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329ecd52a214 namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 2d55fd7a-0180-46d3-af9b-5030872e6fb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:53:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:13Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7329f33a14e49 namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: 5276bf3b-f6b3-4ed8-9ebb-8cc1453ac4fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7331faf7cafb9 namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 832acfe8-4972-4de1-8168-8b240a3772a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a7331faf7df36b namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: f4bc1627-2141-4618-8111-d3f6667b6795 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c1417668-656a-445d-89b2-53c582a08559 kind: Event lastTimestamp: "2026-04-17T16:53:13Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:53:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:13Z" name: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5.18a73320a81ef269 namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: bef3e690-40d7-4f1d-91d7-d5bc9103c266 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8b822-predictor-7b8bf795f7 namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: 8afc34b1-8512-4ff3-9750-8b48dda39960 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: 'Created pod: error-404-isvc-8b822-predictor-7b8bf795f7-l6kw5' metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:54Z" name: error-404-isvc-8b822-predictor-7b8bf795f7.18a7329e87b9f509 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 1fb16f7f-69a0-4ede-92c7-c8afe3f0b3bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: c7c5d37d-6629-49bb-93e1-9e7f9ab0033a kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Scaled up replica set error-404-isvc-8b822-predictor-7b8bf795f7 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:54Z" name: error-404-isvc-8b822-predictor.18a7329e85e27cf8 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: ffc8c9b4-07ff-4676-b515-ab1f5201358e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:44:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 6c8adc82-9a13-454d-9135-b1aae3eec52b kind: Event lastTimestamp: "2026-04-17T16:44:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:54Z" name: error-404-isvc-8b822-predictor.18a732a20456e161 namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: 33a2e419-06b1-4321-b13f-236e2811ce9b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:44:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 6c8adc82-9a13-454d-9135-b1aae3eec52b kind: Event lastTimestamp: "2026-04-17T16:44:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:54Z" name: error-404-isvc-8b822-predictor.18a732a204ba2d76 namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: de02f116-1714-46d2-a9ae-3e83ded15492 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:45:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: 6c8adc82-9a13-454d-9135-b1aae3eec52b kind: Event lastTimestamp: "2026-04-17T16:45:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:24Z" name: error-404-isvc-8b822-predictor.18a732afff93c472 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: f06b10c4-6fc7-43ab-adb7-02a6a381a9f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:45:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: 6c8adc82-9a13-454d-9135-b1aae3eec52b kind: Event lastTimestamp: "2026-04-17T16:45:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:24Z" name: error-404-isvc-8b822-predictor.18a732affffb269a namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 0a196ed9-9768-4786-b768-5f563042817f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b822 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 67dce32f-ac18-4d9d-bf49-3c00d7d4a257 kind: Event lastTimestamp: "2026-04-17T16:44:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:43Z" name: error-404-isvc-8b822.18a7329e85bdbb3b namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 5484ea49-d09c-401a-93c5-2be73119625a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b822 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 67dce32f-ac18-4d9d-bf49-3c00d7d4a257 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: 'Failed to update status for InferenceService "error-404-isvc-8b822": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8b822": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:54Z" name: error-404-isvc-8b822.18a7329e862b7785 namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: 52be1d40-a437-49bd-9b64-ac9eba3a71e2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b822 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 67dce32f-ac18-4d9d-bf49-3c00d7d4a257 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8b822": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:54Z" name: error-404-isvc-8b822.18a7329e862bb973 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 7449b759-0d97-49c1-b822-3a77196c9692 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b822 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 67dce32f-ac18-4d9d-bf49-3c00d7d4a257 kind: Event lastTimestamp: "2026-04-17T16:44:43Z" message: InferenceService [error-404-isvc-8b822] is Ready metadata: creationTimestamp: "2026-04-17T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:43Z" name: error-404-isvc-8b822.18a732a9eecf2464 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: d19fa6a4-87f4-4245-b6cb-462ff29dbddc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:41:44.660069Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14098" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:41:44Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a732804d99cd5c namespace: kserve-ci-e2e-test resourceVersion: "14103" uid: 3b423d0c-1668-4e8f-a436-2a007a53a07a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14107" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:41:45Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:41:45Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a732806684453e namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 88b39744-9f61-4db4-962b-43b77e91e063 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:41:45Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" metadata: creationTimestamp: "2026-04-17T16:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:45Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a7328067e2f9cb namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: 812cd1a1-71de-4e53-8f4d-5835148f2eea reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:41:59Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" in 14.351s (14.351s including waiting). Image size: 1332288608 bytes.' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:59Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73283bf4bef7e namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 7f6cd148-22bc-4d62-afe8-04e5f65c5dd4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:41:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:59Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73283c347f0dc namespace: kserve-ci-e2e-test resourceVersion: "14321" uid: 0180bdd7-e7c2-4b74-b23f-4c8a8ac648fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:41:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:59Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73283c42df2c9 namespace: kserve-ci-e2e-test resourceVersion: "14322" uid: 7aa3c31d-63dc-4fc4-bac7-17c24977bf24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:41:59Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:59Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73283c4414984 namespace: kserve-ci-e2e-test resourceVersion: "14325" uid: 6980297c-b308-4236-b609-3f0527344e7e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:42:03Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.994s (3.994s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-17T16:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:03Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73284b2566987 namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: b93a6669-b6c8-45ac-aa94-fd2433d137fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:42:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:03Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73284b6d97d1b namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: 5f6a0318-8a88-465f-8d67-ec0a2e35ca7c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:42:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:03Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73284b75ed8b4 namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: f841e569-0309-484b-9664-4613299beb7c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:43:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:21Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a732850a9f770c namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: b4b6adca-4e37-4e45-be74-a07be5b20337 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:18Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73296268bcaf2 namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: cd84e22e-6a9e-4e6b-a632-95dfa3a7b48d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:18Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73296268c5458 namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: c9e3da97-2b71-4981-9f0f-a8d5e535cdc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: cca92b8b-ec77-44db-88bc-3cbada2e6604 kind: Event lastTimestamp: "2026-04-17T16:43:21Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:21Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l.18a73296bcbe962e namespace: kserve-ci-e2e-test resourceVersion: "15238" uid: e450a29d-18b9-452a-b093-e56eb5cdf8cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9e0ff-predictor-6655bb8c8f namespace: kserve-ci-e2e-test resourceVersion: "14074" uid: bda88a68-266c-40ed-b50f-24b8190289c1 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: 'Created pod: error-404-isvc-9e0ff-predictor-6655bb8c8f-7m29l' metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:44Z" name: error-404-isvc-9e0ff-predictor-6655bb8c8f.18a732804d2fdf36 namespace: kserve-ci-e2e-test resourceVersion: "14100" uid: 0095926e-1ac7-42da-9701-4bd5e17b0f95 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14072" uid: 90c86811-a73c-415b-ac45-d0beab2ebc70 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Scaled up replica set error-404-isvc-9e0ff-predictor-6655bb8c8f from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:44Z" name: error-404-isvc-9e0ff-predictor.18a73280409a4dc6 namespace: kserve-ci-e2e-test resourceVersion: "14076" uid: b3e7f7dc-f4fa-4dde-9e44-86feaa1f2057 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8b6ab2e9-bca0-4107-909e-28dc198de597 kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: error-404-isvc-9e0ff-predictor.18a73283beca732f namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: 2dcda40b-2bc1-4940-a7db-6b073c0e8dd5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 8b6ab2e9-bca0-4107-909e-28dc198de597 kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: error-404-isvc-9e0ff-predictor.18a73283bf2a7af0 namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: b9592476-53a4-464f-abab-bbe7de1b7fb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14315" uid: 8b6ab2e9-bca0-4107-909e-28dc198de597 kind: Event lastTimestamp: "2026-04-17T16:43:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:14Z" name: error-404-isvc-9e0ff-predictor.18a7328e3b41d364 namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 49de5a0f-74c1-4fcd-8459-23673f6ab21d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14315" uid: 8b6ab2e9-bca0-4107-909e-28dc198de597 kind: Event lastTimestamp: "2026-04-17T16:43:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:14Z" name: error-404-isvc-9e0ff-predictor.18a7328e3b8a5778 namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: 3196eb0e-bf92-488e-8fc3-654bfd16b8d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9e0ff namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: 8b54f683-a754-4378-b729-974bf33b139b kind: Event lastTimestamp: "2026-04-17T16:42:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:42:51Z" name: error-404-isvc-9e0ff.18a73280403585e8 namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: 1cb65674-7be3-4522-8862-62915a5eb551 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9e0ff namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: 8b54f683-a754-4378-b729-974bf33b139b kind: Event lastTimestamp: "2026-04-17T16:42:51Z" message: InferenceService [error-404-isvc-9e0ff] is Ready metadata: creationTimestamp: "2026-04-17T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:42:51Z" name: error-404-isvc-9e0ff.18a7328fc7d8bb9b namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 8f9d7d5d-4d0b-4231-aa20-8ac0f9b5a745 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:52:33.632652Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19578" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:52:33Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a733176761926e namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: acd5df9e-ffdf-4489-b964-28cadacfb2b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ac602-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a7331772f31b0c namespace: kserve-ci-e2e-test resourceVersion: "19594" uid: 523057bc-9118-4008-bc53-bb7b2f29e7be reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:52:34Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a73317a51b9772 namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: bcc951ab-ac5c-49ed-8845-5f03bc129240 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a73317a66ed7ce namespace: kserve-ci-e2e-test resourceVersion: "19617" uid: 1d034d83-9c13-4344-be11-2375f63dcb9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a73317a9caf708 namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: b9bd1ff6-0427-459e-9ebc-5083af3ff5d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a73317aa3f253b namespace: kserve-ci-e2e-test resourceVersion: "19623" uid: 49a8d309-ced4-43a3-986a-09625ea9495f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a73317aa4a7d8e namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: 4124746d-8a45-4fd5-a96a-eb0507572f18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a73317aeb27800 namespace: kserve-ci-e2e-test resourceVersion: "19626" uid: 2a386ae8-5a37-47c4-82aa-fe49d90e0ff9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a73317af37ff33 namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: d66104d9-0656-43f5-89d2-0cf265bba7b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:52:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:53:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:52:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:12Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a7331827355ea2 namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: b3764772-e171-4d64-8c50-9227d344d8ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:53Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a7332a0f7d1d5c namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 5540a183-50e2-4049-b3c3-e74b5a5a0e91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:53Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a7332a0f7d956d namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 535c1198-bceb-4f63-afb4-8a770ddfb211 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 2fd4eaf4-3bae-47ab-853c-dc736ae40134 kind: Event lastTimestamp: "2026-04-17T16:53:58Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-04-17T16:53:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:58Z" name: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w.18a7332b3f100fa8 namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: cfd41e2c-6e0e-4e2d-a097-9c190a3f20e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ac602-predictor-67b8b47b5c namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: 441aae75-bd57-4a8b-8c44-e3763f082f4e kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'Created pod: error-404-isvc-ac602-predictor-67b8b47b5c-d2b2w' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:33Z" name: error-404-isvc-ac602-predictor-67b8b47b5c.18a7331766c0d0db namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: a86e813e-2403-45db-add1-b9b9ad294717 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: fb2df255-475e-4636-aa1a-3147dc987211 kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Scaled up replica set error-404-isvc-ac602-predictor-67b8b47b5c from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:33Z" name: error-404-isvc-ac602-predictor.18a7331765332187 namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: 62b0bf76-1c69-4652-bd03-700d3289a878 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:52:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19569" uid: 1e010c20-9f1f-4f4d-a0ac-58a44945f11d kind: Event lastTimestamp: "2026-04-17T16:53:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:03Z" name: error-404-isvc-ac602-predictor.18a7331ae3936bea namespace: kserve-ci-e2e-test resourceVersion: "19872" uid: ddc0a3d6-67e1-4e08-b9ce-c45ac7a3a673 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:52:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19569" uid: 1e010c20-9f1f-4f4d-a0ac-58a44945f11d kind: Event lastTimestamp: "2026-04-17T16:53:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:03Z" name: error-404-isvc-ac602-predictor.18a7331ae3d4aea9 namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: 0494f2ed-ec70-441e-8174-cd065c293d70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: 1e010c20-9f1f-4f4d-a0ac-58a44945f11d kind: Event lastTimestamp: "2026-04-17T16:53:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:48Z" name: error-404-isvc-ac602-predictor.18a73321e145f386 namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 534c400b-fce5-4e0b-9800-5468898eb209 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: 1e010c20-9f1f-4f4d-a0ac-58a44945f11d kind: Event lastTimestamp: "2026-04-17T16:53:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:48Z" name: error-404-isvc-ac602-predictor.18a73321e18d153e namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: e94749c1-709d-46ca-b2e3-9a0fb42c925b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac602 namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: 5b858f22-fa03-412b-9692-ea09d4aff35f kind: Event lastTimestamp: "2026-04-17T16:53:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:22Z" name: error-404-isvc-ac602.18a7331764f22669 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: 48f6ef0d-eaa7-4790-8b48-bd08d9dcc490 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac602 namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: 5b858f22-fa03-412b-9692-ea09d4aff35f kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-ac602": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ac602": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:52:33Z" name: error-404-isvc-ac602.18a733176595148b namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: 795edc97-b609-4519-ac79-726e0a48dcf2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac602 namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: 5b858f22-fa03-412b-9692-ea09d4aff35f kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ac602": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:52:33Z" name: error-404-isvc-ac602.18a7331765954a86 namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: d217791d-bc98-403e-9bbd-9c2caf7ee0d4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac602 namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: 5b858f22-fa03-412b-9692-ea09d4aff35f kind: Event lastTimestamp: "2026-04-17T16:53:22Z" message: InferenceService [error-404-isvc-ac602] is Ready metadata: creationTimestamp: "2026-04-17T16:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:22Z" name: error-404-isvc-ac602.18a73322e2316e92 namespace: kserve-ci-e2e-test resourceVersion: "20178" uid: 87310a49-9542-42ce-a746-34bdcafae6c2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:53:53.828694Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:53:53Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a7332a137004cb namespace: kserve-ci-e2e-test resourceVersion: "20574" uid: 41c00e94-bbc9-4529-b5b3-7a5ae427dc64 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:53:54Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a7332a2d3ee5d0 namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 2bab1638-3d72-43ee-999b-d377d35ed13e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a7332a2f8f0c76 namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: 2069cc1b-fa74-4741-84b5-7567cb84f968 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a7332a31fb4286 namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 2e84cd02-a92a-411a-a1c0-9985867a42b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a7332a326cdad7 namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: 5dce674e-e9e8-4f4d-a48c-25999b19f10c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a7332a3279fc20 namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: f3931e63-9027-4ff5-91f7-466f9387831a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a7332a3528017b namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: da278c87-594a-47c2-97bf-5fa0765abd8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a7332a3596bf04 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 5b5f2115-c7aa-406e-ac11-b2771f1f6a53 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-17T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T17:03:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:22Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a7332a60905dd1 namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: eb7fa7f2-a5fc-4a7a-aa98-c7484c5d95fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:18Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a733ada1f0804c namespace: kserve-ci-e2e-test resourceVersion: "25001" uid: b9a50137-7626-48ec-bab7-f21ab98643fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:18Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a733ada1f18d52 namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: 4a7a3967-8ef7-43a5-a2ae-6f1476463a59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: bfd7ff5e-fc98-4023-8ffc-c410bedc4ae1 kind: Event lastTimestamp: "2026-04-17T17:03:22Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:03:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:22Z" name: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq.18a733ae64ce5b27 namespace: kserve-ci-e2e-test resourceVersion: "25078" uid: 453cac30-94bf-4b80-8794-71fc96e0b44b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c68dd-predictor-9956c4bc9 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: 7911c448-8447-4207-b677-8f5e4f0fd70e kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: 'Created pod: error-404-isvc-c68dd-predictor-9956c4bc9-5pwwq' metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:53Z" name: error-404-isvc-c68dd-predictor-9956c4bc9.18a7332a12fe6ff4 namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: c8bf0e00-b6c3-4d19-a638-0c91cd65cb20 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: 9f91a455-aa80-460c-9572-597068d1c395 kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: Scaled up replica set error-404-isvc-c68dd-predictor-9956c4bc9 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:53Z" name: error-404-isvc-c68dd-predictor.18a7332a1147def7 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 02481829-3750-4db6-97ad-aa9cf06cc426 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: 70bf3055-f076-45ee-8323-3a26fb905345 kind: Event lastTimestamp: "2026-04-17T16:54:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:38Z" name: error-404-isvc-c68dd-predictor.18a7332d8faff26c namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 4abecf7e-3484-4b85-98da-71e282d618c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: 70bf3055-f076-45ee-8323-3a26fb905345 kind: Event lastTimestamp: "2026-04-17T16:54:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:38Z" name: error-404-isvc-c68dd-predictor.18a7332d8ff6b5cd namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: d297a77a-2b56-4fdc-ab4b-a09003276dbc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:54:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 70bf3055-f076-45ee-8323-3a26fb905345 kind: Event lastTimestamp: "2026-04-17T16:55:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:08Z" name: error-404-isvc-c68dd-predictor.18a733380c37ee2a namespace: kserve-ci-e2e-test resourceVersion: "21478" uid: dc1db70e-2ff6-4670-8fd3-87a143926308 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:54:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 70bf3055-f076-45ee-8323-3a26fb905345 kind: Event lastTimestamp: "2026-04-17T16:55:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:08Z" name: error-404-isvc-c68dd-predictor.18a733380ca39a9e namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: ea52990a-a14c-4ba2-90f7-ade587bfe971 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c68dd namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 79efee17-ef5e-4146-b7e1-f11a429bc936 kind: Event lastTimestamp: "2026-04-17T16:54:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:42Z" name: error-404-isvc-c68dd.18a7332a10fceeba namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: fbfef324-f2bd-49d2-9695-d3228cbbc1a3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c68dd namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 79efee17-ef5e-4146-b7e1-f11a429bc936 kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: 'Failed to update status for InferenceService "error-404-isvc-c68dd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c68dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:53Z" name: error-404-isvc-c68dd.18a7332a1157b70e namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: 3037ed6b-1a42-416f-8811-862bcc2d73d5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c68dd namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 79efee17-ef5e-4146-b7e1-f11a429bc936 kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c68dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:53Z" name: error-404-isvc-c68dd.18a7332a1157fa42 namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 9841d7c1-823c-4f72-a533-083d52e3c941 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c68dd namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 79efee17-ef5e-4146-b7e1-f11a429bc936 kind: Event lastTimestamp: "2026-04-17T16:54:42Z" message: InferenceService [error-404-isvc-c68dd] is Ready metadata: creationTimestamp: "2026-04-17T16:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:42Z" name: error-404-isvc-c68dd.18a7333556e89df5 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 092ff990-3560-45e3-a309-0c16c687fd23 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:43:18.575159Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:43:18Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a732962b604f91 namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: c6be3c9e-7863-42a3-a4d6-946dde41ed67 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:43:19Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a732964547db4e namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: 86756150-a0e7-4c41-ab55-f228780ebbae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a7329646a90be8 namespace: kserve-ci-e2e-test resourceVersion: "15206" uid: e9df9852-ae5a-4f40-8b8f-f68fdd5f5a27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a7329649dd444c namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 32aeab2c-0f02-4912-a17a-7b4858b578ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a732964a53b75e namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 1398eed7-3304-4e09-bf42-88a80a2fb077 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a732964a5e6fed namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: b4a218d0-a224-4481-8bad-23367c4beaa1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a732964deca262 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: 2017e78a-2acd-498b-9cae-008774995d62 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a732964e59d2b8 namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0ae9f144-5855-406a-8a3d-e9ecbc308374 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:43:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:52:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:36Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a732968ea66f94 namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: d359f4cb-48fd-4f26-9c73-6563ddce1b77 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a7331762d13cd3 namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: d0da7167-fba0-430e-aa3c-50adbdd6dbc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a7331762d1b071 namespace: kserve-ci-e2e-test resourceVersion: "19559" uid: 93c5bf6d-9528-4992-b2ed-01d0f959480d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: c530e539-fe55-47db-ae30-3447fd68f304 kind: Event lastTimestamp: "2026-04-17T16:52:36Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:52:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:36Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh.18a7331802fb9c08 namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: 1603cef0-c1ec-4fd0-a756-ddaa428cda87 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ce4d4-predictor-5d96695c7c namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 92602d8c-dbc6-48a4-be7f-38b225982c66 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: 'Created pod: error-404-isvc-ce4d4-predictor-5d96695c7c-7bsvh' metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:18Z" name: error-404-isvc-ce4d4-predictor-5d96695c7c.18a732962aeaff4f namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: 58e17a0c-8c19-4169-9216-173057fce5c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 1bf30001-3a50-4733-bcb9-066bdc57fb99 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: Scaled up replica set error-404-isvc-ce4d4-predictor-5d96695c7c from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:18Z" name: error-404-isvc-ce4d4-predictor.18a7329629405759 namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: acb83c91-c20c-4d45-827e-1346249fdb5a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 5f7743a3-607f-4b2f-897a-38132669067d kind: Event lastTimestamp: "2026-04-17T16:44:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:33Z" name: error-404-isvc-ce4d4-predictor.18a73299a79f8f9f namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: 12851f64-22b0-4573-93b0-d4a678087432 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 5f7743a3-607f-4b2f-897a-38132669067d kind: Event lastTimestamp: "2026-04-17T16:44:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:33Z" name: error-404-isvc-ce4d4-predictor.18a73299a81b550d namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 5d842068-d2d0-4cd4-aa76-b1433e9285cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:44:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 5f7743a3-607f-4b2f-897a-38132669067d kind: Event lastTimestamp: "2026-04-17T16:44:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:18Z" name: error-404-isvc-ce4d4-predictor.18a732a0a660f577 namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: e3c63609-435b-408c-bc77-77189fcd2321 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:44:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 5f7743a3-607f-4b2f-897a-38132669067d kind: Event lastTimestamp: "2026-04-17T16:44:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:18Z" name: error-404-isvc-ce4d4-predictor.18a732a0a6c622fb namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: 46dd5a88-d273-4e91-b131-b99d753f792e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 8533b619-bb9e-4737-a8c2-adb40e190a51 kind: Event lastTimestamp: "2026-04-17T16:44:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:06Z" name: error-404-isvc-ce4d4.18a73296290905e6 namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: f036c44e-7644-49d4-a80e-bdf097b72d95 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 8533b619-bb9e-4737-a8c2-adb40e190a51 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: 'Failed to update status for InferenceService "error-404-isvc-ce4d4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ce4d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:18Z" name: error-404-isvc-ce4d4.18a7329629557ed2 namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: 2e526190-99c0-40d4-9dc2-5e0731bc11b4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 8533b619-bb9e-4737-a8c2-adb40e190a51 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ce4d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:18Z" name: error-404-isvc-ce4d4.18a732962955b4aa namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: a1fb6830-2b47-4179-a5b8-7c2b97fa24e3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 8533b619-bb9e-4737-a8c2-adb40e190a51 kind: Event lastTimestamp: "2026-04-17T16:44:06Z" message: InferenceService [error-404-isvc-ce4d4] is Ready metadata: creationTimestamp: "2026-04-17T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:06Z" name: error-404-isvc-ce4d4.18a732a149c8289b namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 62c9e941-5d49-4966-bf58-f3e2abfcef7d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:53:09.270943Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d0d22-predictor-76db56b8f-hz2ck to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:53:09Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7331fb396bae4 namespace: kserve-ci-e2e-test resourceVersion: "20010" uid: 4dc1ffdd-04dd-490a-b08a-d7290668af54 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d0d22-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7331fbd2ba76b namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: 2d5072a9-e290-4415-9cc1-63d3f10bce03 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:53:10Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:53:10Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7331ff0cb2978 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 9a139c8e-ecc5-48a8-ac27-e2dd8c1b9409 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:53:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:10Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7331ff20146ed namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 409483f7-b653-4988-b82d-791f199283e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:53:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:10Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7331ff5ae1e01 namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: 45273710-588d-4e03-81cd-1470669ccdbd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:53:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:10Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7331ff6232b5e namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 89053eb3-185c-401d-87ae-570000dcb9b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:53:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:10Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7331ff62f8b52 namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 33985930-db65-43d0-935a-e0c7e8d9cbae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:53:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:10Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7331ff8f16b4a namespace: kserve-ci-e2e-test resourceVersion: "20053" uid: 81de103a-032f-4d1c-8ba6-e811761cbabf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:53:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:10Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7331ff979f4ec namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 430c3cb5-677f-4860-8232-cd5ae3079d70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-17T16:53:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:53:57Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:57Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7332018a7a337 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: c847274b-db81-4079-8bcc-6c6773be492a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:29Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a73332600f8d1a namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 58803efc-4019-4864-9c89-fe27d5996523 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: e0f6cda1-d31a-47bf-bca2-34508ccf7d7e kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:29Z" name: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck.18a7333260100c65 namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: 692cc06d-6f51-4b52-bee8-73a688504a5f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d0d22-predictor-76db56b8f namespace: kserve-ci-e2e-test resourceVersion: "19999" uid: 9c7f15c1-c16a-4761-b94d-285060baf246 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: 'Created pod: error-404-isvc-d0d22-predictor-76db56b8f-hz2ck' metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:09Z" name: error-404-isvc-d0d22-predictor-76db56b8f.18a7331fb32a89cf namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: 08364a79-a113-4848-b9cb-7692a06265be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "19997" uid: be351627-fb28-422a-9b77-ff3788700573 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Scaled up replica set error-404-isvc-d0d22-predictor-76db56b8f from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:09Z" name: error-404-isvc-d0d22-predictor.18a7331fb1a8332f namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: cac763cd-0803-425e-8739-8a1d044871d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:53:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: 5fe1b9fa-217f-4025-993c-e6477050b38d kind: Event lastTimestamp: "2026-04-17T16:54:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:24Z" name: error-404-isvc-d0d22-predictor.18a7332330287eb2 namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: fa7b095a-66d4-42d1-84bc-5a15a1673ec0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:53:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: 5fe1b9fa-217f-4025-993c-e6477050b38d kind: Event lastTimestamp: "2026-04-17T16:54:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:24Z" name: error-404-isvc-d0d22-predictor.18a73323309f69dd namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: e5f34b42-9f86-4112-9f86-c538c6a08d50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: 5fe1b9fa-217f-4025-993c-e6477050b38d kind: Event lastTimestamp: "2026-04-17T16:54:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:09Z" name: error-404-isvc-d0d22-predictor.18a73326af4e9795 namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: 842ee339-bfeb-4d3f-9767-54044296f7bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: 5fe1b9fa-217f-4025-993c-e6477050b38d kind: Event lastTimestamp: "2026-04-17T16:54:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:09Z" name: error-404-isvc-d0d22-predictor.18a73326afb1e132 namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 3f65c656-c595-428e-bf4f-11ddc190c440 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 0540062c-b7d5-4392-827b-4cc47c1034b2 kind: Event lastTimestamp: "2026-04-17T16:54:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:08Z" name: error-404-isvc-d0d22.18a7331fb17edeb9 namespace: kserve-ci-e2e-test resourceVersion: "20751" uid: a5e88d63-cd97-41f2-ac9e-ded138509807 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 0540062c-b7d5-4392-827b-4cc47c1034b2 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-d0d22": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d0d22": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:09Z" name: error-404-isvc-d0d22.18a7331fb20279a8 namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: e071dcb9-c0fc-48a6-9369-f505e7c258ae reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 0540062c-b7d5-4392-827b-4cc47c1034b2 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d0d22": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:09Z" name: error-404-isvc-d0d22.18a7331fb202b879 namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: 00bb73e2-b1d9-4c72-a8f4-1fb5d1fef014 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: 0540062c-b7d5-4392-827b-4cc47c1034b2 kind: Event lastTimestamp: "2026-04-17T16:54:08Z" message: InferenceService [error-404-isvc-d0d22] is Ready metadata: creationTimestamp: "2026-04-17T16:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:08Z" name: error-404-isvc-d0d22.18a7332d6374af01 namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 3070ba20-2492-4c46-b0ef-fb987da7f43c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:54:29.529998Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f6018-predictor-6c99ffbf99-pllwl to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:54:29Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a733326366a94e namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 1359574a-6383-4d1c-b45c-583d1a3235b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:54:29Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a733327d0f5c9b namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 973d1b9d-ee06-4dc2-8d23-37bf9d6cadfb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:29Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a733327e56b069 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 3b2689c4-dafd-4c87-a426-6e2f602e6dec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a733328208be14 namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 7037c664-9c82-4559-b1c5-bb423d4dc6cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a73332828fdf1b namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 46251513-ead7-400a-a3d9-03144bf134cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a73332829c46ff namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: b6365ce6-2e49-4d07-b0a9-d61f5f661514 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a733328565fc05 namespace: kserve-ci-e2e-test resourceVersion: "21090" uid: 5ce4eb72-2795-427a-95e7-197ae1ff660b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a7333285d23379 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: df8e2449-c1eb-4654-a9f7-50850304168c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T17:03:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:47Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a73332c9cdebeb namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: f4495d6e-a06b-4d7a-a59e-73b5dde02e80 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:44Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a733b38c4c5292 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 5a8c7a71-09f3-47c3-a389-3945bdcf1dea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:44Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a733b38c4ce7e3 namespace: kserve-ci-e2e-test resourceVersion: "25456" uid: 5a5a7d74-2946-48cb-a439-78b29319da13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: a7a3aa06-adda-4fdb-b593-939a28b6c765 kind: Event lastTimestamp: "2026-04-17T17:03:47Z" message: 'Readiness probe failed: Get "https://10.132.0.40:8643/healthz": dial tcp 10.132.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:03:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:47Z" name: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl.18a733b43e2f75c4 namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 2753eb28-60d3-4f73-876f-9589955ea911 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f6018-predictor-6c99ffbf99 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: 92e569e6-06ea-49c6-ab72-d0202bf698b8 kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: 'Created pod: error-404-isvc-f6018-predictor-6c99ffbf99-pllwl' metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:29Z" name: error-404-isvc-f6018-predictor-6c99ffbf99.18a7333262d983f9 namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: c27e6062-e55e-4e14-923e-c6fb3332354d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: b973355b-2820-4f9b-969d-de477c06c09b kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: Scaled up replica set error-404-isvc-f6018-predictor-6c99ffbf99 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:29Z" name: error-404-isvc-f6018-predictor.18a73332612f2f0b namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 99b51f7b-0750-418c-939f-8ecb56e00f04 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 3f0c078d-9735-43c8-ab01-d271c166f22a kind: Event lastTimestamp: "2026-04-17T16:55:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:44Z" name: error-404-isvc-f6018-predictor.18a73335df9deaca namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: 785df28d-9b85-4d8d-a27b-4cfcc03b6b73 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 3f0c078d-9735-43c8-ab01-d271c166f22a kind: Event lastTimestamp: "2026-04-17T16:55:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:44Z" name: error-404-isvc-f6018-predictor.18a73335dffd510a namespace: kserve-ci-e2e-test resourceVersion: "21814" uid: 3d87ca6e-ee32-4093-8e88-40e373851314 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: 3f0c078d-9735-43c8-ab01-d271c166f22a kind: Event lastTimestamp: "2026-04-17T16:54:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:59Z" name: error-404-isvc-f6018-predictor.18a733395eabd7e5 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 2e00c25d-2a74-4b9a-8e33-89122fdec51a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: 3f0c078d-9735-43c8-ab01-d271c166f22a kind: Event lastTimestamp: "2026-04-17T16:54:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:59Z" name: error-404-isvc-f6018-predictor.18a733395f0c3026 namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: b3c1edee-a493-4187-859a-6bdb4a4724d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6018 namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: c853957a-790b-450f-b07f-010c0b9088c5 kind: Event lastTimestamp: "2026-04-17T16:55:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:55:17Z" name: error-404-isvc-f6018.18a733326104218b namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 8a9a188b-5f12-4118-a3fd-3251b8668182 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6018 namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: c853957a-790b-450f-b07f-010c0b9088c5 kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-f6018": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f6018": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:29Z" name: error-404-isvc-f6018.18a7333261580b40 namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: b8db6651-0771-4447-940c-45e6e70cf864 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6018 namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: c853957a-790b-450f-b07f-010c0b9088c5 kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f6018": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:29Z" name: error-404-isvc-f6018.18a7333261585b15 namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: bd520de5-af71-4900-9c1a-4501ee74940c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6018 namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: c853957a-790b-450f-b07f-010c0b9088c5 kind: Event lastTimestamp: "2026-04-17T16:55:17Z" message: InferenceService [error-404-isvc-f6018] is Ready metadata: creationTimestamp: "2026-04-17T16:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:55:17Z" name: error-404-isvc-f6018.18a7333d84d31a77 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: 839886bd-ed73-45a4-a3ef-65450437d7ac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:41:44.074515Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f to ip-10-0-129-144.ec2.internal metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:41:44Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732802ab2f3a9 namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: b74e65ed-d025-466d-9932-5d2cbe6a701a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14018" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:41:44Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a73280448b6139 namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: b7e86728-210c-44fb-b420-2631e7786fdb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:44Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a7328045f40d69 namespace: kserve-ci-e2e-test resourceVersion: "14091" uid: fe85bf7a-4a94-4cee-b908-c203635bac9f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:41:48Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" in 4.132s (4.132s including waiting). Image size: 301288360 bytes.' metadata: creationTimestamp: "2026-04-17T16:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:48Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732813c42c64e namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 919d982d-e3db-458b-a6b9-9c4eba59ded2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:41:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:48Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732813f85189e namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 92c818fd-3f0f-4db1-a121-2ce0cc962060 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:41:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:48Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732813ff46715 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 263c0c53-04eb-42f9-a00b-48ca95947f0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:41:52Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-17T16:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:52Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a73282154a4fdf namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: e5484dd9-c948-43f8-ac58-5fc87c91952c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:42:05Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 12.919s (12.919s including waiting). Image size: 1560612266 bytes.' metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:05Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a73285175cf408 namespace: kserve-ci-e2e-test resourceVersion: "14381" uid: 9d4d1a10-65ec-4e3a-872a-11786cda769e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:42:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:05Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732851b66aadc namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: 343ba728-d189-4f1f-9e33-6489b3672c10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:42:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:05Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732851bd9e642 namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 53c0420e-9ce4-4692-a11e-908248934fd0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:42:05Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:05Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732851be64e34 namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: 78f82691-6e46-4c7a-a9ee-7a992805a817 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.109s (2.109s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a7328599a47834 namespace: kserve-ci-e2e-test resourceVersion: "14406" uid: 529e986b-e32d-46bc-863b-94bf32c088f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732859c894711 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 0f20f687-decf-420d-8e81-46a9c933a088 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732859cf71fab namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: 7564c830-c359-4693-b877-66a841f279c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:42:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a732860e517c73 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 7b323304-3666-4597-a99b-16ff32fa410a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a7329e841d2361 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: b5b67897-00c9-4e21-8761-17ec1fbfa1ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a7329e841d93be namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 9419ab5b-870d-432d-8622-a5dab4dca475 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 8a67d317-a529-4322-97ba-9f8fb27dc2b7 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f.18a7329ebc8ba4ed namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: f51ec8cf-6804-434d-854b-83e86e0302b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-144.ec2.internal source: component: kubelet host: ip-10-0-129-144.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-5b497dcd98 namespace: kserve-ci-e2e-test resourceVersion: "13996" uid: bc149e63-b8db-405b-9828-ac08d00861f6 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-5b497dcd98-gqt9f' metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:44Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98.18a732802a4e08d4 namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: d3b13697-e14a-45fc-ba8b-1594b11cad52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13995" uid: 02189cc6-7509-4ca2-83e0-b8145c15aa0c kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-5b497dcd98 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:43Z" name: isvc-sklearn-graph-1-predictor.18a7328024ea9488 namespace: kserve-ci-e2e-test resourceVersion: "13998" uid: 6024ce0c-4e15-4868-97a5-f4fa7c7fbc1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: 8a3df504-61e4-4db6-9f88-417dd6d160c2 kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: isvc-sklearn-graph-1-predictor.18a73283aa72ed56 namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: 55b7e896-f3aa-41d1-9f78-f7b198d6c362 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: 8a3df504-61e4-4db6-9f88-417dd6d160c2 kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: isvc-sklearn-graph-1-predictor.18a73283aab81b81 namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: 56063701-19e8-4818-8535-845cd079eaae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: 8a3df504-61e4-4db6-9f88-417dd6d160c2 kind: Event lastTimestamp: "2026-04-17T16:43:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:29Z" name: isvc-sklearn-graph-1-predictor.18a7328e272f13b5 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: f3d8217e-be47-4f94-a2a2-ff1c8ea1170f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: 8a3df504-61e4-4db6-9f88-417dd6d160c2 kind: Event lastTimestamp: "2026-04-17T16:43:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:29Z" name: isvc-sklearn-graph-1-predictor.18a7328e2790137b namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 253cdced-9ece-4530-a615-22f2504d6f6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: 5b6ce042-aadb-48ef-a895-0cc4113210b4 kind: Event lastTimestamp: "2026-04-17T16:43:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:15Z" name: isvc-sklearn-graph-1.18a732802b16e01f namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: fdb031c7-31ed-4472-aaf4-815774479ea3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: 5b6ce042-aadb-48ef-a895-0cc4113210b4 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:41:44Z" name: isvc-sklearn-graph-1.18a732802ba01745 namespace: kserve-ci-e2e-test resourceVersion: "14016" uid: a61be2f5-5fd7-4632-9da9-6c716f68e66d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: 5b6ce042-aadb-48ef-a895-0cc4113210b4 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:41:44Z" name: isvc-sklearn-graph-1.18a732802ba07aed namespace: kserve-ci-e2e-test resourceVersion: "14017" uid: 7d186828-8865-4493-8f02-3788fecf46db reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 5b6ce042-aadb-48ef-a895-0cc4113210b4 kind: Event lastTimestamp: "2026-04-17T16:43:15Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-17T16:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:15Z" name: isvc-sklearn-graph-1.18a7329570ed122b namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: 8976d8ee-1ab4-40c0-bbf7-39c91b8375fd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:41:45.352681Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14132" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl to ip-10-0-138-170.ec2.internal metadata: creationTimestamp: "2026-04-17T16:41:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:41:45Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a7328076e23916 namespace: kserve-ci-e2e-test resourceVersion: "14136" uid: 7d26faaa-e8b9-412f-b312-3119a615df98 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:41:45Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:41:45Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a73280900659a1 namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: abcca992-8aab-418c-989c-37b66dbd32f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:41:45Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" metadata: creationTimestamp: "2026-04-17T16:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:45Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a73280915c28fe namespace: kserve-ci-e2e-test resourceVersion: "14156" uid: 7e13c159-c669-4c42-b2ff-bdf36c29991f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:41:49Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" in 3.251s (3.251s including waiting). Image size: 301288360 bytes.' metadata: creationTimestamp: "2026-04-17T16:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:49Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a732815323797c namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: fbc98591-70be-4cf5-bc8b-21f658cd7c97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:41:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:49Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a732815621de8e namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 2477cbd0-5b65-40b3-a57b-a9c247a4c7eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:41:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:49Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a732815696d85b namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: eaa19d0e-dd86-4936-9970-6f830dd4dd2c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:41:53Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-17T16:41:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:53Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a7328249526d02 namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: 916b4547-a270-463b-8c26-0fb53e01abf8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:42:05Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 12.425s (12.425s including waiting). Image size: 1560612266 bytes.' metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:05Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a732852de9a429 namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: 78826527-4f95-42c8-8fe2-28f7f5cd7597 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:42:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:05Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a7328531438da6 namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: 3a34533b-fa3c-43b2-ad9e-613045c19c02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:42:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:05Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a7328531cd6503 namespace: kserve-ci-e2e-test resourceVersion: "14393" uid: 60d2ac9a-f367-44eb-8558-3b51028c77bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:42:05Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:05Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a7328531dab886 namespace: kserve-ci-e2e-test resourceVersion: "14394" uid: 17c6e407-c13d-4490-8b08-c8c0e113b694 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.127s (2.127s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a73285b0aa70a5 namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: a231afb6-0ad1-45d7-89c9-cf7a2ba9756a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a73285b3b65077 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: beed53bc-ab2f-43f5-8074-b0d86f525ebf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a73285b42c5ea0 namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: dc18508c-ad1d-4b6f-8bd3-e5e6049bb2f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:42:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a7328606750ebc namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 901c1420-badd-4a7f-9c57-f3ef14f1d446 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a7329e7f6aef68 namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: 781dc29f-aff2-412c-8a02-53fe3af65197 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a7329e7f6bb6a5 namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: d86afcdd-060f-455a-879c-70d9f010d28a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: a947f5ca-ee30-454a-8725-075f92f19468 kind: Event lastTimestamp: "2026-04-17T16:43:55Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:55Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl.18a7329eb4b46654 namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: fca4021e-859e-4fec-a5de-06627c2dabae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-170.ec2.internal source: component: kubelet host: ip-10-0-138-170.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-847f6cf74b namespace: kserve-ci-e2e-test resourceVersion: "14119" uid: 44b58d9f-9ac6-48ba-9c5d-5ca1a9b0a5e2 kind: Event lastTimestamp: "2026-04-17T16:41:45Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-847f6cf74b-88pwl' metadata: creationTimestamp: "2026-04-17T16:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:45Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b.18a73280765d9ee6 namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: cb5ac25a-a05b-443d-a62c-66589eaad822 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 04de2aa7-37d8-4f6c-a3ef-983fe08d04ba kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-847f6cf74b from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:44Z" name: isvc-sklearn-graph-2-predictor.18a732805c6e43d2 namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: c1ff0837-39e7-40da-8564-7ea7530b5d83 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14118" uid: 061d663d-a954-4f61-a004-0e4ef9c8fd67 kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: isvc-sklearn-graph-2-predictor.18a73283db06dcd1 namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: f10948e0-c759-4f9d-8d82-9af76a8c2f23 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14118" uid: 061d663d-a954-4f61-a004-0e4ef9c8fd67 kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: isvc-sklearn-graph-2-predictor.18a73283db6c17bc namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: ec75e506-28b6-4af2-a742-4dd4fc6687ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14328" uid: 061d663d-a954-4f61-a004-0e4ef9c8fd67 kind: Event lastTimestamp: "2026-04-17T16:43:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:29Z" name: isvc-sklearn-graph-2-predictor.18a7328e5781c40f namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: fb0f8e3f-4bba-4273-ae4c-0139c5995cac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14328" uid: 061d663d-a954-4f61-a004-0e4ef9c8fd67 kind: Event lastTimestamp: "2026-04-17T16:43:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:30Z" name: isvc-sklearn-graph-2-predictor.18a7328e57e599b1 namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: 4a553d37-7805-49ec-abf2-0afcb65e82f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14092" uid: 100e896c-6c07-4b90-8dcf-1779621fdaa2 kind: Event lastTimestamp: "2026-04-17T16:43:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:15Z" name: isvc-sklearn-graph-2.18a732805c5d6bc4 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: ec89f763-70bf-40cc-853c-6fc6665f438d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 100e896c-6c07-4b90-8dcf-1779621fdaa2 kind: Event lastTimestamp: "2026-04-17T16:43:15Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-17T16:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:15Z" name: isvc-sklearn-graph-2.18a7329569078443 namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: 3bc4b967-6fef-4853-8bd3-48b0b94e6c44 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:41:44.480507Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-qgmdv to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:41:44Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a7328042e5dc31 namespace: kserve-ci-e2e-test resourceVersion: "14083" uid: c999a812-4d7d-4752-8a48-cf198caa727d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:44Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a7328050d927a7 namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 5a291308-12f3-4d65-a9d6-64d2a32a1744 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14088" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:41:45Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:41:45Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a73280834c1f67 namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 7e612d09-968f-4915-ab72-f6db497f1f1a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:41:45Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" metadata: creationTimestamp: "2026-04-17T16:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:45Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a7328084f8fde6 namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: cfb054c5-6c06-47de-975e-06ef55112e81 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:41:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" in 5.276s (5.276s including waiting). Image size: 301288360 bytes.' metadata: creationTimestamp: "2026-04-17T16:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:50Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a73281bf7679b4 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: f43050ba-4b18-4e14-bad5-d539a5262700 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:41:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:50Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a73281c3854a10 namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 624c7cff-562d-49d9-835a-61e503f576d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:41:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:50Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a73281c40f403b namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 2072e0a7-14ca-4f84-a412-b65abb1e9886 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:41:55Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-17T16:41:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:55Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a73282effe1257 namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 872a82f1-9f82-4300-9400-8aa2bdd857f6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:42:18Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 22.822s (22.822s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-17T16:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:18Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a732884056fade namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: 70643de4-9e6b-458a-80f9-e4fb78d3cdc4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:42:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:18Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a7328846e59e48 namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: cbabe559-ed14-4948-9ff9-356c1ba4d4cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:42:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:18Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a7328848852e47 namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: 61cbdac9-42f1-4d92-8257-f3c0517d0e6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:42:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:18Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a73288489289b8 namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: 066a973c-5811-4505-89bb-ae595d2beb6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:42:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:19Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a732884e78e332 namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: 6ee87266-e095-484f-af01-d582100cd259 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:42:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:19Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a732884fbb7e77 namespace: kserve-ci-e2e-test resourceVersion: "14524" uid: ad5a34ff-0265-4b2b-96b6-ffcbabdad661 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:42:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:43:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a732888b917dec namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: adc5e313-de43-44c3-8f40-f980fed0a1ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a7329e7bfd2329 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: ec10ec6d-d386-4c8c-bb3a-e05e81299f9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a7329e7bfe580b namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: fe54cf61-5582-4eeb-a52f-697ee09a0ba6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 2806963c-78df-401c-b5e5-ab8166c86d7f kind: Event lastTimestamp: "2026-04-17T16:43:56Z" message: 'Readiness probe failed: Get "https://10.132.0.21:8643/healthz": dial tcp 10.132.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-qgmdv.18a7329ee5b590d6 namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: f590c4eb-ede2-48fc-8528-1a3737a75323 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "14057" uid: 9a4aeaf7-cc67-47b4-804f-e9d664d3683a kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-qgmdv' metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:44Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a732804202a8af namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: 29a9bb66-4dae-4b52-8116-67398d4044ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: 39b5feb3-50e2-4e42-be0b-584ff3ab0bd2 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:44Z" name: isvc-xgboost-graph-predictor.18a7328035c42d74 namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: b87f71d1-0610-4c50-9286-a7774f0cd130 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14055" uid: d805a574-e55d-4912-a6e2-e9e268d6ac1c kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: isvc-xgboost-graph-predictor.18a73283b433d5d9 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 380db2f0-c223-4a5e-9ab5-a9fe7856a12d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14055" uid: d805a574-e55d-4912-a6e2-e9e268d6ac1c kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: isvc-xgboost-graph-predictor.18a73283b4966113 namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: 1d28cf30-3002-4b1d-ab09-84e41e77f910 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: d805a574-e55d-4912-a6e2-e9e268d6ac1c kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:44Z" name: isvc-xgboost-graph-predictor.18a7328e30a8d710 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: c5e1191d-a98c-4642-9a8e-9bd055baa540 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: d805a574-e55d-4912-a6e2-e9e268d6ac1c kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:44Z" name: isvc-xgboost-graph-predictor.18a7328e310e3f4c namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 56355112-e37a-4677-9728-f249b2410e0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14036" uid: 2ca21f1d-a9a8-4e26-95e2-fc01003cd580 kind: Event lastTimestamp: "2026-04-17T16:43:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:26Z" name: isvc-xgboost-graph.18a73280359e07fb namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: c8811d1e-5884-446d-98d7-78616d715373 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: 2ca21f1d-a9a8-4e26-95e2-fc01003cd580 kind: Event lastTimestamp: "2026-04-17T16:43:26Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-17T16:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:26Z" name: isvc-xgboost-graph.18a73297eea10d38 namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: 815dddaf-3194-4756-80de-971c9c993d1c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:43:44.028510Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-86b99c7c88-lngx8 namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 671eac1a-ae21-4662-83f9-a14856f158a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-86b99c7c88-lngx8 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:43:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:43:44Z" name: model-chainer-86b99c7c88-lngx8.18a7329c1883b1cd namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 966de7e2-9aa9-4aab-a1be-1370346300ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:44Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-86b99c7c88-lngx8 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 671eac1a-ae21-4662-83f9-a14856f158a1 kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:43:44Z" name: model-chainer-86b99c7c88-lngx8.18a7329c3194fdc0 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: 5181433b-db76-4e20-b3a6-f01aaeb78322 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-86b99c7c88-lngx8 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 671eac1a-ae21-4662-83f9-a14856f158a1 kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:44Z" name: model-chainer-86b99c7c88-lngx8.18a7329c32bbb85c namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: fa41a5a7-54e0-4ea6-8aed-7d76004a22e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-86b99c7c88-lngx8 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 671eac1a-ae21-4662-83f9-a14856f158a1 kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-17T16:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:44Z" name: model-chainer-86b99c7c88-lngx8.18a7329c35e45409 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 1d8dbc6b-da2c-4e43-a2eb-f0c2722a5805 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-86b99c7c88-lngx8 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 671eac1a-ae21-4662-83f9-a14856f158a1 kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-17T16:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:44Z" name: model-chainer-86b99c7c88-lngx8.18a7329c365b5352 namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: 50335f51-679a-45df-bdfd-85d377b45634 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-86b99c7c88-lngx8 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 671eac1a-ae21-4662-83f9-a14856f158a1 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: model-chainer-86b99c7c88-lngx8.18a7329e77d704ce namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: d11e5e62-f2ff-4c0e-b225-44e9484aeecf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:43:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-86b99c7c88-lngx8 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 671eac1a-ae21-4662-83f9-a14856f158a1 kind: Event lastTimestamp: "2026-04-17T16:44:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:44:21Z" name: model-chainer-86b99c7c88-lngx8.18a7329ef5a7a1ef namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: a4e963ef-16ff-44f4-9382-cf9dafc5ac7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-86b99c7c88 namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 5e1ba0a8-24fc-4544-ab84-832e82653779 kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: 'Created pod: model-chainer-86b99c7c88-lngx8' metadata: creationTimestamp: "2026-04-17T16:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:44Z" name: model-chainer-86b99c7c88.18a7329c1816a130 namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: 97f98806-351c-40e7-8455-05f310080c0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: ad3f7b10-470b-4449-bbc6-c3858cf0174a kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: Scaled up replica set model-chainer-86b99c7c88 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:44Z" name: model-chainer.18a7329c170ebe82 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 1cebabdc-04cc-46c0-8960-a763077ba362 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: d95a21d0-2cdb-4ad9-bc87-0c58733d262a kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-17T16:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:44Z" name: model-chainer.18a7329c1c9287ad namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 085945ad-21d9-4117-98bc-003a064b0fa2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:44:54.368168Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8b822-89658595d-trzvx namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: 174c6520-09a4-48e5-a2b4-4ed9e1d3f556 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8b822-89658595d-trzvx to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:44:54Z" name: sequence-graph-8b822-89658595d-trzvx.18a732ac7915ba46 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: d462f7b3-396d-4e39-a168-35620d9b61c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:54Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8b822-89658595d-trzvx namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 174c6520-09a4-48e5-a2b4-4ed9e1d3f556 kind: Event lastTimestamp: "2026-04-17T16:44:54Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:44:54Z" name: sequence-graph-8b822-89658595d-trzvx.18a732ac93686682 namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 25be245e-3850-4424-8443-f933ed499089 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b822} kind: Pod name: sequence-graph-8b822-89658595d-trzvx namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 174c6520-09a4-48e5-a2b4-4ed9e1d3f556 kind: Event lastTimestamp: "2026-04-17T16:44:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:44:54Z" name: sequence-graph-8b822-89658595d-trzvx.18a732ac94a6ec2d namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: ce052add-c8aa-409e-aaf6-b8a56c74c2e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b822} kind: Pod name: sequence-graph-8b822-89658595d-trzvx namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 174c6520-09a4-48e5-a2b4-4ed9e1d3f556 kind: Event lastTimestamp: "2026-04-17T16:44:54Z" message: 'Created container: sequence-graph-8b822' metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:44:54Z" name: sequence-graph-8b822-89658595d-trzvx.18a732ac9786def4 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: 43ff19f2-49fc-4b8b-9064-0e92257a93d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b822} kind: Pod name: sequence-graph-8b822-89658595d-trzvx namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 174c6520-09a4-48e5-a2b4-4ed9e1d3f556 kind: Event lastTimestamp: "2026-04-17T16:44:54Z" message: Started container sequence-graph-8b822 metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:44:54Z" name: sequence-graph-8b822-89658595d-trzvx.18a732ac97f56ca7 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: c91935f0-a1e9-4c2c-8c5c-79e9bcb1dec3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b822} kind: Pod name: sequence-graph-8b822-89658595d-trzvx namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 174c6520-09a4-48e5-a2b4-4ed9e1d3f556 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Stopping container sequence-graph-8b822 metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: sequence-graph-8b822-89658595d-trzvx.18a7331fa51e864d namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: 6f21714a-8bf3-4e64-a43d-f71bab333844 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b822} kind: Pod name: sequence-graph-8b822-89658595d-trzvx namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 174c6520-09a4-48e5-a2b4-4ed9e1d3f556 kind: Event lastTimestamp: "2026-04-17T16:53:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:36Z" name: sequence-graph-8b822-89658595d-trzvx.18a733203bf6f486 namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 1690ecaa-d58d-4119-85dd-107fe649fa5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8b822-89658595d namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: de440c95-16da-4f73-917e-e23d5439f0c5 kind: Event lastTimestamp: "2026-04-17T16:44:54Z" message: 'Created pod: sequence-graph-8b822-89658595d-trzvx' metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:54Z" name: sequence-graph-8b822-89658595d.18a732ac788b8a67 namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: a34db9b7-e3e4-4471-ba65-a2f016d4f1bc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8b822 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 9b592698-fe6f-4cf5-bc8b-b24f49559456 kind: Event lastTimestamp: "2026-04-17T16:44:54Z" message: Scaled up replica set sequence-graph-8b822-89658595d from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:54Z" name: sequence-graph-8b822.18a732ac7771883c namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 6d1ac24a-f97b-4d84-ba4d-16ad9b144d6c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8b822 namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 173c5c35-a06d-42a9-8af8-0de697bb838c kind: Event lastTimestamp: "2026-04-17T16:44:54Z" message: InferenceGraph [sequence-graph-8b822] is Ready metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:54Z" name: sequence-graph-8b822.18a732ac7d137c90 namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: 7e6c7be4-5a3c-4668-9116-e7f9bf04408f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8b822 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 14485f3d-56e4-48dc-a978-438fbae03366 kind: Event lastTimestamp: "2026-04-17T16:45:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:09Z" name: sequence-graph-8b822.18a732aff5eb5a4b namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: ffc89609-dad3-4d79-bcaa-bfc4e3c07976 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8b822 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 14485f3d-56e4-48dc-a978-438fbae03366 kind: Event lastTimestamp: "2026-04-17T16:45:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:09Z" name: sequence-graph-8b822.18a732aff64c3dbe namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 7d85b728-296b-4237-8881-a47646e6bf01 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8b822 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 14485f3d-56e4-48dc-a978-438fbae03366 kind: Event lastTimestamp: "2026-04-17T16:45:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:24Z" name: sequence-graph-8b822.18a732b37505eebb namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 3d5673b9-b16b-4558-8b3a-62efab42476e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8b822 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 14485f3d-56e4-48dc-a978-438fbae03366 kind: Event lastTimestamp: "2026-04-17T16:45:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:24Z" name: sequence-graph-8b822.18a732b37556b2b6 namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: b436b169-a0cd-471a-8b07-cab79a36e68d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:54:19.204196Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d0d22-77bb574b5d-lkw4b namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 7c296e6d-7831-43eb-84f7-bfb00b09fee8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-d0d22-77bb574b5d-lkw4b to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:54:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:54:19Z" name: sequence-graph-d0d22-77bb574b5d-lkw4b.18a7332ffbef8198 namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: f1790f6a-5142-4719-8a40-50ace30f1a42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:19Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d0d22-77bb574b5d-lkw4b namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 7c296e6d-7831-43eb-84f7-bfb00b09fee8 kind: Event lastTimestamp: "2026-04-17T16:54:19Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:54:19Z" name: sequence-graph-d0d22-77bb574b5d-lkw4b.18a733301519ec4f namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: ceb63c52-ff7e-432f-b711-def1392863cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d0d22} kind: Pod name: sequence-graph-d0d22-77bb574b5d-lkw4b namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 7c296e6d-7831-43eb-84f7-bfb00b09fee8 kind: Event lastTimestamp: "2026-04-17T16:54:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T16:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:19Z" name: sequence-graph-d0d22-77bb574b5d-lkw4b.18a7333016bb03af namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: b2a475bf-0e7a-4794-b9d7-462b3c632ba8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d0d22} kind: Pod name: sequence-graph-d0d22-77bb574b5d-lkw4b namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 7c296e6d-7831-43eb-84f7-bfb00b09fee8 kind: Event lastTimestamp: "2026-04-17T16:54:19Z" message: 'Created container: sequence-graph-d0d22' metadata: creationTimestamp: "2026-04-17T16:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:19Z" name: sequence-graph-d0d22-77bb574b5d-lkw4b.18a733301a3d5520 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 86e6d4cb-76aa-47bb-902e-fc3b3b0e511a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d0d22} kind: Pod name: sequence-graph-d0d22-77bb574b5d-lkw4b namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 7c296e6d-7831-43eb-84f7-bfb00b09fee8 kind: Event lastTimestamp: "2026-04-17T16:54:19Z" message: Started container sequence-graph-d0d22 metadata: creationTimestamp: "2026-04-17T16:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:19Z" name: sequence-graph-d0d22-77bb574b5d-lkw4b.18a733301aac2660 namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: 3a1e1992-be02-4aba-916c-474b3810c1fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d0d22} kind: Pod name: sequence-graph-d0d22-77bb574b5d-lkw4b namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 7c296e6d-7831-43eb-84f7-bfb00b09fee8 kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: Stopping container sequence-graph-d0d22 metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:29Z" name: sequence-graph-d0d22-77bb574b5d-lkw4b.18a7333254c900b9 namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 6c08f75e-09a8-4eee-bf7b-aed75176a003 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d0d22} kind: Pod name: sequence-graph-d0d22-77bb574b5d-lkw4b namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 7c296e6d-7831-43eb-84f7-bfb00b09fee8 kind: Event lastTimestamp: "2026-04-17T16:54:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:56Z" name: sequence-graph-d0d22-77bb574b5d-lkw4b.18a73332c7cf0ad1 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: c45c6d48-d552-4e9c-84a3-3b0fc116baad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-d0d22-77bb574b5d namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: 0fcc4e7d-42e2-4c86-9132-880d886582eb kind: Event lastTimestamp: "2026-04-17T16:54:19Z" message: 'Created pod: sequence-graph-d0d22-77bb574b5d-lkw4b' metadata: creationTimestamp: "2026-04-17T16:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:19Z" name: sequence-graph-d0d22-77bb574b5d.18a7332ffb566e08 namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: e7bec952-6ef7-40af-b60c-512193a138ef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: af51ae12-8e9f-408f-8301-198b98a3ac9b kind: Event lastTimestamp: "2026-04-17T16:54:19Z" message: Scaled up replica set sequence-graph-d0d22-77bb574b5d from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:19Z" name: sequence-graph-d0d22.18a7332ffa8b2ac4 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 547eb751-b2d8-4b9e-b22a-2845472d4f23 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: b3ebbe84-6cf3-430a-b3c9-a49fce30e88f kind: Event lastTimestamp: "2026-04-17T16:54:19Z" message: InferenceGraph [sequence-graph-d0d22] is Ready metadata: creationTimestamp: "2026-04-17T16:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:19Z" name: sequence-graph-d0d22.18a7332fffcb3e96 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 7915dd1d-5a02-4255-8ab3-64792ef3c1d0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:55:29.492797Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f6018-777885f489-bxxmg namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: d0c2def3-ae1a-4e26-9457-c3814b0f6898 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f6018-777885f489-bxxmg to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:55:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:55:29Z" name: sequence-graph-f6018-777885f489-bxxmg.18a7334059766c7f namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 958edae2-3935-4351-b0a6-091cf935a82a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f6018-777885f489-bxxmg namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: d0c2def3-ae1a-4e26-9457-c3814b0f6898 kind: Event lastTimestamp: "2026-04-17T16:55:29Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:55:29Z" name: sequence-graph-f6018-777885f489-bxxmg.18a7334072cb5529 namespace: kserve-ci-e2e-test resourceVersion: "21690" uid: 09934693-6780-4113-a26d-075ebffffef5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6018} kind: Pod name: sequence-graph-f6018-777885f489-bxxmg namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: d0c2def3-ae1a-4e26-9457-c3814b0f6898 kind: Event lastTimestamp: "2026-04-17T16:55:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T16:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:55:29Z" name: sequence-graph-f6018-777885f489-bxxmg.18a733407401ee92 namespace: kserve-ci-e2e-test resourceVersion: "21692" uid: 3bedf54f-371c-4e27-a807-a0cc15e4cc18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6018} kind: Pod name: sequence-graph-f6018-777885f489-bxxmg namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: d0c2def3-ae1a-4e26-9457-c3814b0f6898 kind: Event lastTimestamp: "2026-04-17T16:55:29Z" message: 'Created container: sequence-graph-f6018' metadata: creationTimestamp: "2026-04-17T16:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:55:29Z" name: sequence-graph-f6018-777885f489-bxxmg.18a7334076e30b91 namespace: kserve-ci-e2e-test resourceVersion: "21696" uid: ed8fce05-7841-4c68-bfa9-900684c1fd54 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6018} kind: Pod name: sequence-graph-f6018-777885f489-bxxmg namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: d0c2def3-ae1a-4e26-9457-c3814b0f6898 kind: Event lastTimestamp: "2026-04-17T16:55:29Z" message: Started container sequence-graph-f6018 metadata: creationTimestamp: "2026-04-17T16:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:55:29Z" name: sequence-graph-f6018-777885f489-bxxmg.18a7334077671dde namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 662ad7d7-0a9d-401c-b0b4-2ea7d47ba725 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6018} kind: Pod name: sequence-graph-f6018-777885f489-bxxmg namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: d0c2def3-ae1a-4e26-9457-c3814b0f6898 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: Stopping container sequence-graph-f6018 metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:44Z" name: sequence-graph-f6018-777885f489-bxxmg.18a733b387e7e6ae namespace: kserve-ci-e2e-test resourceVersion: "25436" uid: 2c55f1fc-7ba6-46dc-b902-b7f5e059893b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:03:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6018} kind: Pod name: sequence-graph-f6018-777885f489-bxxmg namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: d0c2def3-ae1a-4e26-9457-c3814b0f6898 kind: Event lastTimestamp: "2026-04-17T17:04:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:03:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:11Z" name: sequence-graph-f6018-777885f489-bxxmg.18a733b40de5673f namespace: kserve-ci-e2e-test resourceVersion: "25930" uid: 6fcc3fd6-32b4-4fc8-96d3-5aa80ba00284 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f6018-777885f489 namespace: kserve-ci-e2e-test resourceVersion: "21667" uid: d16d8813-41d1-4c3c-9798-9dc3e96d3975 kind: Event lastTimestamp: "2026-04-17T16:55:29Z" message: 'Created pod: sequence-graph-f6018-777885f489-bxxmg' metadata: creationTimestamp: "2026-04-17T16:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:29Z" name: sequence-graph-f6018-777885f489.18a7334058e4eae4 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: c7acad7d-5a89-4f54-b10f-e139dd3e8869 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f6018 namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 4197ae88-1a2e-405a-b64b-8d199118feea kind: Event lastTimestamp: "2026-04-17T16:55:29Z" message: Scaled up replica set sequence-graph-f6018-777885f489 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:29Z" name: sequence-graph-f6018.18a7334058545604 namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 64337319-95b8-4ce3-9ba5-8f80f61ff32f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f6018 namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: f8ef15a1-daa7-44e8-b8fb-ecde7ee4c495 kind: Event lastTimestamp: "2026-04-17T16:55:29Z" message: InferenceGraph [sequence-graph-f6018] is Ready metadata: creationTimestamp: "2026-04-17T16:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:55:29Z" name: sequence-graph-f6018.18a733405c0ba929 namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: c09e3824-52db-4531-a8d7-31a7a5fde156 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:55:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f6018 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: 1546fb61-ad5a-4f18-b5aa-f863183763c1 kind: Event lastTimestamp: "2026-04-17T16:55:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:59Z" name: sequence-graph-f6018.18a73343d6ba0a87 namespace: kserve-ci-e2e-test resourceVersion: "21920" uid: 1c92044d-090c-4cc4-99b7-fdedfaf5d58e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:55:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f6018 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: 1546fb61-ad5a-4f18-b5aa-f863183763c1 kind: Event lastTimestamp: "2026-04-17T16:55:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:59Z" name: sequence-graph-f6018.18a73343d6fd2590 namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: 00bf5962-5fd0-4c64-9a19-ef4c53b5255e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:04:18.838152Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-41c4d-654b49775c-n9pd5 namespace: kserve-ci-e2e-test resourceVersion: "26005" uid: 87459e4c-4eb7-49f2-b1c7-9c3653c7d39d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-41c4d-654b49775c-n9pd5 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T17:04:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:04:18Z" name: splitter-graph-41c4d-654b49775c-n9pd5.18a733bb98e7800c namespace: kserve-ci-e2e-test resourceVersion: "26012" uid: 87003a3a-fae0-471b-b149-be1282711403 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:18Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-41c4d-654b49775c-n9pd5 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 87459e4c-4eb7-49f2-b1c7-9c3653c7d39d kind: Event lastTimestamp: "2026-04-17T17:04:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-41c4d-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:18Z" name: splitter-graph-41c4d-654b49775c-n9pd5.18a733bba229bf44 namespace: kserve-ci-e2e-test resourceVersion: "26025" uid: 2d201c33-1bbc-4310-a23f-e4d7717df1cd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:19Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-41c4d-654b49775c-n9pd5 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: 87459e4c-4eb7-49f2-b1c7-9c3653c7d39d kind: Event lastTimestamp: "2026-04-17T17:04:19Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:04:19Z" name: splitter-graph-41c4d-654b49775c-n9pd5.18a733bbd5c7b034 namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: fb1f6fb6-7afe-4fa7-b0ad-098765e11627 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c4d} kind: Pod name: splitter-graph-41c4d-654b49775c-n9pd5 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 87459e4c-4eb7-49f2-b1c7-9c3653c7d39d kind: Event lastTimestamp: "2026-04-17T17:04:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:19Z" name: splitter-graph-41c4d-654b49775c-n9pd5.18a733bbd6f9cb77 namespace: kserve-ci-e2e-test resourceVersion: "26038" uid: 13423acf-4e0e-446a-9116-2ba097b945f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c4d} kind: Pod name: splitter-graph-41c4d-654b49775c-n9pd5 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 87459e4c-4eb7-49f2-b1c7-9c3653c7d39d kind: Event lastTimestamp: "2026-04-17T17:04:19Z" message: 'Created container: splitter-graph-41c4d' metadata: creationTimestamp: "2026-04-17T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:19Z" name: splitter-graph-41c4d-654b49775c-n9pd5.18a733bbda0de1a8 namespace: kserve-ci-e2e-test resourceVersion: "26039" uid: 4754f1cc-02e6-4687-8030-7e751f1c9a2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c4d} kind: Pod name: splitter-graph-41c4d-654b49775c-n9pd5 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 87459e4c-4eb7-49f2-b1c7-9c3653c7d39d kind: Event lastTimestamp: "2026-04-17T17:04:19Z" message: Started container splitter-graph-41c4d metadata: creationTimestamp: "2026-04-17T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:19Z" name: splitter-graph-41c4d-654b49775c-n9pd5.18a733bbda7b4bf4 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: de6ff0ed-9385-44e9-a4ba-43f69ce009a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c4d} kind: Pod name: splitter-graph-41c4d-654b49775c-n9pd5 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 87459e4c-4eb7-49f2-b1c7-9c3653c7d39d kind: Event lastTimestamp: "2026-04-17T17:04:28Z" message: Stopping container splitter-graph-41c4d metadata: creationTimestamp: "2026-04-17T17:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:28Z" name: splitter-graph-41c4d-654b49775c-n9pd5.18a733bdf1ad730e namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: d7958ba3-e8d2-438d-9634-64915c691bc0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c4d} kind: Pod name: splitter-graph-41c4d-654b49775c-n9pd5 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 87459e4c-4eb7-49f2-b1c7-9c3653c7d39d kind: Event lastTimestamp: "2026-04-17T17:04:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:55Z" name: splitter-graph-41c4d-654b49775c-n9pd5.18a733be6cd6409c namespace: kserve-ci-e2e-test resourceVersion: "26528" uid: c0dea6c2-4651-4a99-b5d8-fad599010fb5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-41c4d-654b49775c namespace: kserve-ci-e2e-test resourceVersion: "26002" uid: 47b7d998-373a-4f12-a5f1-3f46211723cc kind: Event lastTimestamp: "2026-04-17T17:04:18Z" message: 'Created pod: splitter-graph-41c4d-654b49775c-n9pd5' metadata: creationTimestamp: "2026-04-17T17:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:18Z" name: splitter-graph-41c4d-654b49775c.18a733bb9856c20a namespace: kserve-ci-e2e-test resourceVersion: "26007" uid: 9c44afa5-bfb1-4c9d-81c0-a8559a82d051 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-41c4d namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 6cf12595-b090-454b-91b2-c48724d049ed kind: Event lastTimestamp: "2026-04-17T17:04:18Z" message: Scaled up replica set splitter-graph-41c4d-654b49775c from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:18Z" name: splitter-graph-41c4d.18a733bb97a3c0b0 namespace: kserve-ci-e2e-test resourceVersion: "26004" uid: 3a16120a-6032-4edd-bf59-13ff7e654d3c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-41c4d namespace: kserve-ci-e2e-test resourceVersion: "26018" uid: a463a6eb-38b9-474d-86ff-e1355c53dda4 kind: Event lastTimestamp: "2026-04-17T17:04:18Z" message: InferenceGraph [splitter-graph-41c4d] is Ready metadata: creationTimestamp: "2026-04-17T17:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:18Z" name: splitter-graph-41c4d.18a733bb9b4a12b2 namespace: kserve-ci-e2e-test resourceVersion: "26020" uid: aee1029b-9bb3-4db5-8679-1050d828c67a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:05:39.129717Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7afa6-7fb6d7495d-5g5l4 namespace: kserve-ci-e2e-test resourceVersion: "26894" uid: fd6d8cec-e328-49cc-a077-8d0b71e06a76 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-7afa6-7fb6d7495d-5g5l4 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T17:05:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:05:39Z" name: splitter-graph-7afa6-7fb6d7495d-5g5l4.18a733ce4aa79746 namespace: kserve-ci-e2e-test resourceVersion: "26899" uid: fb2d3b40-88c9-40aa-9264-04ff5aaf89ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7afa6-7fb6d7495d-5g5l4 namespace: kserve-ci-e2e-test resourceVersion: "26903" uid: fd6d8cec-e328-49cc-a077-8d0b71e06a76 kind: Event lastTimestamp: "2026-04-17T17:05:39Z" message: Add eth0 [10.132.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:05:39Z" name: splitter-graph-7afa6-7fb6d7495d-5g5l4.18a733ce6395fbd3 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: dca8e210-07bb-4e87-a5a3-812664b8d112 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7afa6} kind: Pod name: splitter-graph-7afa6-7fb6d7495d-5g5l4 namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: fd6d8cec-e328-49cc-a077-8d0b71e06a76 kind: Event lastTimestamp: "2026-04-17T17:05:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:05:39Z" name: splitter-graph-7afa6-7fb6d7495d-5g5l4.18a733ce64d2a3e2 namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: d76635cf-d9a6-4915-ba44-254ab870e728 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7afa6} kind: Pod name: splitter-graph-7afa6-7fb6d7495d-5g5l4 namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: fd6d8cec-e328-49cc-a077-8d0b71e06a76 kind: Event lastTimestamp: "2026-04-17T17:05:39Z" message: 'Created container: splitter-graph-7afa6' metadata: creationTimestamp: "2026-04-17T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:05:39Z" name: splitter-graph-7afa6-7fb6d7495d-5g5l4.18a733ce67cd3e88 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 35cd9f36-0382-4932-90db-1b2fc029858a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7afa6} kind: Pod name: splitter-graph-7afa6-7fb6d7495d-5g5l4 namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: fd6d8cec-e328-49cc-a077-8d0b71e06a76 kind: Event lastTimestamp: "2026-04-17T17:05:39Z" message: Started container splitter-graph-7afa6 metadata: creationTimestamp: "2026-04-17T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:05:39Z" name: splitter-graph-7afa6-7fb6d7495d-5g5l4.18a733ce683ff3d1 namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: f098da4b-cf41-439d-ae1f-236b5ec2da0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7afa6} kind: Pod name: splitter-graph-7afa6-7fb6d7495d-5g5l4 namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: fd6d8cec-e328-49cc-a077-8d0b71e06a76 kind: Event lastTimestamp: "2026-04-17T17:13:53Z" message: Stopping container splitter-graph-7afa6 metadata: creationTimestamp: "2026-04-17T17:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:13:53Z" name: splitter-graph-7afa6-7fb6d7495d-5g5l4.18a734417aadfecb namespace: kserve-ci-e2e-test resourceVersion: "30394" uid: 895be06d-1f53-4303-8ba4-0478805d8625 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7afa6} kind: Pod name: splitter-graph-7afa6-7fb6d7495d-5g5l4 namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: fd6d8cec-e328-49cc-a077-8d0b71e06a76 kind: Event lastTimestamp: "2026-04-17T17:14:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:14:21Z" name: splitter-graph-7afa6-7fb6d7495d-5g5l4.18a7344208a5c395 namespace: kserve-ci-e2e-test resourceVersion: "30624" uid: 9a675cab-40cb-4ab3-9d35-aa0ed291278c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-7afa6-7fb6d7495d namespace: kserve-ci-e2e-test resourceVersion: "26891" uid: d1443624-60c6-4aff-bc80-a9d351a576db kind: Event lastTimestamp: "2026-04-17T17:05:39Z" message: 'Created pod: splitter-graph-7afa6-7fb6d7495d-5g5l4' metadata: creationTimestamp: "2026-04-17T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:39Z" name: splitter-graph-7afa6-7fb6d7495d.18a733ce4a414167 namespace: kserve-ci-e2e-test resourceVersion: "26896" uid: 89b1d4b4-a925-435b-8cae-37ce30faa7c8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: a8cf57c0-89c7-458d-90ec-77d1821ada28 kind: Event lastTimestamp: "2026-04-17T17:05:39Z" message: Scaled up replica set splitter-graph-7afa6-7fb6d7495d from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:39Z" name: splitter-graph-7afa6.18a733ce494b45c9 namespace: kserve-ci-e2e-test resourceVersion: "26892" uid: 5f118f6f-799b-4bb1-ac3a-c41f85438784 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26905" uid: f46f50e5-4578-4963-9ae7-5a20ad58e8f1 kind: Event lastTimestamp: "2026-04-17T17:05:39Z" message: InferenceGraph [splitter-graph-7afa6] is Ready metadata: creationTimestamp: "2026-04-17T17:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:05:39Z" name: splitter-graph-7afa6.18a733ce4e1820c9 namespace: kserve-ci-e2e-test resourceVersion: "26907" uid: f66d9977-f567-44cb-ac1c-eb2fc7d5bba7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: 4a958b37-ba7b-4c4e-b19a-596d5e8ccad1 kind: Event lastTimestamp: "2026-04-17T17:05:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:05:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:54Z" name: splitter-graph-7afa6.18a733d1c80e6489 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: e84162c3-70ef-46b4-84f9-3ff0742fa499 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: 4a958b37-ba7b-4c4e-b19a-596d5e8ccad1 kind: Event lastTimestamp: "2026-04-17T17:05:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:05:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:54Z" name: splitter-graph-7afa6.18a733d1c86ca203 namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 5e43d661-9aab-4e39-bdb2-93d03ae1f5f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:06:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: 4a958b37-ba7b-4c4e-b19a-596d5e8ccad1 kind: Event lastTimestamp: "2026-04-17T17:06:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:06:09Z" name: splitter-graph-7afa6.18a733d5470fb280 namespace: kserve-ci-e2e-test resourceVersion: "27137" uid: 7a3506ab-b2eb-4a77-a757-d879bb2237d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:06:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: 4a958b37-ba7b-4c4e-b19a-596d5e8ccad1 kind: Event lastTimestamp: "2026-04-17T17:06:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:06:09Z" name: splitter-graph-7afa6.18a733d547702b0c namespace: kserve-ci-e2e-test resourceVersion: "27139" uid: b5ec6d2c-e7e5-4886-a6a4-9bca31e58a61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:03:18.841165Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24983" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:03:18Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733ada0ce2232 namespace: kserve-ci-e2e-test resourceVersion: "24993" uid: 0d147006-7418-44d4-9a24-3aa805801551 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "25006" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:03:19Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733adb9d9bd56 namespace: kserve-ci-e2e-test resourceVersion: "25030" uid: 5cb64ec4-10ad-4296-9f8f-025be97688d5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733adbb5bc93c namespace: kserve-ci-e2e-test resourceVersion: "25032" uid: 762d1578-6a26-4a0a-be48-7b5634a7023d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733adbfc70cf1 namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 2c187166-6c5e-497e-8a77-094093925fd2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733adc03c32e9 namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: 1774d3f7-f334-47d4-b2a3-7acd1bde10dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733adc05d9e38 namespace: kserve-ci-e2e-test resourceVersion: "25038" uid: 2afef17e-88e4-4066-b1f6-71233c62f669 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733adc2903d6c namespace: kserve-ci-e2e-test resourceVersion: "25040" uid: ae31a25e-4d46-4eca-9c64-3539a6001670 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:03:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:19Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733adc2fff299 namespace: kserve-ci-e2e-test resourceVersion: "25042" uid: 42f8896a-5c31-4e71-88b6-0afc4ecce0e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:03:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:56Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733ae13df65ed namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: f34c6c3e-a849-4622-ba20-e5b1462d6f39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:29Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733bdf7b89e45 namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: e7f05564-8106-41ea-8f4f-8a5ec85164d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:29Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733bdf7b93e99 namespace: kserve-ci-e2e-test resourceVersion: "26157" uid: fb447ab6-736d-4f92-b15e-68e28d48b976 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 286f4108-4887-46df-87ae-8c5cdb92a6e6 kind: Event lastTimestamp: "2026-04-17T17:04:31Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:31Z" name: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp.18a733be9bf6f010 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: 417a3bb1-3829-4e1f-a60f-6f12102ea43c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-41c4d-predictor-56f84dd86b namespace: kserve-ci-e2e-test resourceVersion: "24961" uid: cd562bfa-d212-4324-a7cf-afd6c702922a kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: 'Created pod: success-200-isvc-41c4d-predictor-56f84dd86b-fzdzp' metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:18Z" name: success-200-isvc-41c4d-predictor-56f84dd86b.18a733ada049f289 namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 9bb9e3fb-67ce-46f2-841f-7777f6c32217 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "24956" uid: c96ca859-6b77-4c78-844f-6016cd49cb81 kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: Scaled up replica set success-200-isvc-41c4d-predictor-56f84dd86b from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:18Z" name: success-200-isvc-41c4d-predictor.18a733ad9bcb3dd3 namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: 1ff5288f-7a09-49d9-b78e-6d4f169b4c13 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "24960" uid: 590ed622-c3f9-4623-acb2-5d697b030708 kind: Event lastTimestamp: "2026-04-17T17:03:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:33Z" name: success-200-isvc-41c4d-predictor.18a733b11a60a6bd namespace: kserve-ci-e2e-test resourceVersion: "25177" uid: 4d6ab527-3a2c-4d64-897d-ba71b1fb12bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "24960" uid: 590ed622-c3f9-4623-acb2-5d697b030708 kind: Event lastTimestamp: "2026-04-17T17:03:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:33Z" name: success-200-isvc-41c4d-predictor.18a733b11ae684d1 namespace: kserve-ci-e2e-test resourceVersion: "25179" uid: 68e1a443-e2e4-4ad1-a25b-8b4b3dcbc459 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:03:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: 590ed622-c3f9-4623-acb2-5d697b030708 kind: Event lastTimestamp: "2026-04-17T17:04:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:18Z" name: success-200-isvc-41c4d-predictor.18a733b499db4836 namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: 445944ca-5ff0-4c79-a79b-68d4d32b330b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:03:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-41c4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: 590ed622-c3f9-4623-acb2-5d697b030708 kind: Event lastTimestamp: "2026-04-17T17:04:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:18Z" name: success-200-isvc-41c4d-predictor.18a733b49a1e1494 namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 290354d3-b358-4830-82d4-24e14faa6224 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-41c4d namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: 4a7661ee-a11c-425b-9bd0-31de07c3a519 kind: Event lastTimestamp: "2026-04-17T17:04:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:06Z" name: success-200-isvc-41c4d.18a733ad9bbfee13 namespace: kserve-ci-e2e-test resourceVersion: "25898" uid: 8e5b1ff5-f422-4b88-92fe-90179556c7c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-41c4d namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: 4a7661ee-a11c-425b-9bd0-31de07c3a519 kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: 'Failed to update status for InferenceService "success-200-isvc-41c4d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-41c4d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:03:18Z" name: success-200-isvc-41c4d.18a733ad9c78b68d namespace: kserve-ci-e2e-test resourceVersion: "24969" uid: 00a99b75-74d5-4575-b943-360cff8d54f8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-41c4d namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: 4a7661ee-a11c-425b-9bd0-31de07c3a519 kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-41c4d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:03:18Z" name: success-200-isvc-41c4d.18a733ad9c78fed1 namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: 040ec76d-6b04-4edb-b88e-05955f00a0c5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-41c4d namespace: kserve-ci-e2e-test resourceVersion: "25895" uid: 4a7661ee-a11c-425b-9bd0-31de07c3a519 kind: Event lastTimestamp: "2026-04-17T17:04:06Z" message: InferenceService [success-200-isvc-41c4d] is Ready metadata: creationTimestamp: "2026-04-17T17:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:06Z" name: success-200-isvc-41c4d.18a733b8d161a53b namespace: kserve-ci-e2e-test resourceVersion: "25896" uid: 3820ab27-fb1e-4552-8460-12f8fe86538d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:03:44.350565Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25487" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:03:44Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b39148b9a4 namespace: kserve-ci-e2e-test resourceVersion: "25491" uid: e648dcaa-47f7-4091-b0ff-2a6c84f9c656 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6941b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:44Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b3a08781c9 namespace: kserve-ci-e2e-test resourceVersion: "25526" uid: 73459dfe-3aaa-4016-b5d3-85af59948589 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25497" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:03:45Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b3ce61c0e3 namespace: kserve-ci-e2e-test resourceVersion: "25566" uid: da6381a9-69da-41ac-89c1-f69c56f4629a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b3cfc8f6c3 namespace: kserve-ci-e2e-test resourceVersion: "25569" uid: 876cc868-3cc5-4015-9535-747e542f86e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b3d319716f namespace: kserve-ci-e2e-test resourceVersion: "25570" uid: bcc10646-9f8e-4190-a63d-aa68101c62f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b3d3903f4d namespace: kserve-ci-e2e-test resourceVersion: "25571" uid: 818693d0-23b5-497f-8033-2c30930e85c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b3d39df87e namespace: kserve-ci-e2e-test resourceVersion: "25572" uid: ad3dd294-d609-4f7c-991d-b9f2936a1d7e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b3d7d4a5f0 namespace: kserve-ci-e2e-test resourceVersion: "25577" uid: be0736dd-1c73-445c-bc9d-234aa858437a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:03:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:45Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b3d848cb6a namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 253bf2b3-9d7e-4eec-a38f-d11d1df58a48 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T17:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:04:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:31Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a733b3eb29014a namespace: kserve-ci-e2e-test resourceVersion: "26278" uid: 42287b88-9ad5-4daa-9416-7c3096f5a876 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:21:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:21:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:21:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:21:13Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a734a7f2cd89a2 namespace: kserve-ci-e2e-test resourceVersion: "33478" uid: 31f147a5-0e75-4d66-994b-8ede7aeba1c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:21:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: b227b01e-aa15-4737-8813-9e1a7021a599 kind: Event lastTimestamp: "2026-04-17T17:21:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:21:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:21:13Z" name: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz.18a734a7f2ce83fc namespace: kserve-ci-e2e-test resourceVersion: "33479" uid: c469dd76-d69b-4645-b262-42b8039cac11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6941b-predictor-5648f5ddc4 namespace: kserve-ci-e2e-test resourceVersion: "25475" uid: 7cfa820e-72be-492e-bfa5-bb24b1fe8543 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: 'Created pod: success-200-isvc-6941b-predictor-5648f5ddc4-hkwmz' metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:44Z" name: success-200-isvc-6941b-predictor-5648f5ddc4.18a733b390d56f06 namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: d19b2b41-6f3b-4701-86c3-122c50b17188 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25470" uid: 3eb2ee52-ee9f-47a0-b24b-27cdeec7f095 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: Scaled up replica set success-200-isvc-6941b-predictor-5648f5ddc4 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:03:44Z" name: success-200-isvc-6941b-predictor.18a733b38eed57a0 namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 2f2bac5a-2ec3-495f-aec4-5f4c4ade9f47 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: ff0706f2-f931-43f8-a4c5-4bd09138c8b1 kind: Event lastTimestamp: "2026-04-17T17:05:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:14Z" name: success-200-isvc-6941b-predictor.18a733b70d771cbe namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 4b34a918-c090-4604-95cb-f50918e82063 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: ff0706f2-f931-43f8-a4c5-4bd09138c8b1 kind: Event lastTimestamp: "2026-04-17T17:05:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:14Z" name: success-200-isvc-6941b-predictor.18a733b70de0b872 namespace: kserve-ci-e2e-test resourceVersion: "26687" uid: 2f727ece-414b-4b19-a46d-ccb17bf1affc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: ff0706f2-f931-43f8-a4c5-4bd09138c8b1 kind: Event lastTimestamp: "2026-04-17T17:04:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:14Z" name: success-200-isvc-6941b-predictor.18a733ba8cc1bf4b namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: d6eedc85-a231-4f88-8977-31148a1a8a12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6941b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: ff0706f2-f931-43f8-a4c5-4bd09138c8b1 kind: Event lastTimestamp: "2026-04-17T17:04:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:14Z" name: success-200-isvc-6941b-predictor.18a733ba8d08668c namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a4cb278b-3ac1-48e7-80c6-136a7b950bc0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "25444" uid: 0821c6c0-0b73-4ecd-9e98-ecdddd48be95 kind: Event lastTimestamp: "2026-04-17T17:04:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:42Z" name: success-200-isvc-6941b.18a733b38ecd19bb namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: 2a14465a-5852-428e-8469-1c0e114f20b3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "25444" uid: 0821c6c0-0b73-4ecd-9e98-ecdddd48be95 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-6941b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6941b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:03:44Z" name: success-200-isvc-6941b.18a733b38f753750 namespace: kserve-ci-e2e-test resourceVersion: "25482" uid: e02af631-ed9b-4e8a-b8e0-e0c81a4b0a0b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "25444" uid: 0821c6c0-0b73-4ecd-9e98-ecdddd48be95 kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6941b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:03:44Z" name: success-200-isvc-6941b.18a733b38f75a67c namespace: kserve-ci-e2e-test resourceVersion: "25484" uid: 5cb03669-4c1a-4810-9661-a590b6fbb7a0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6941b namespace: kserve-ci-e2e-test resourceVersion: "26378" uid: 0821c6c0-0b73-4ecd-9e98-ecdddd48be95 kind: Event lastTimestamp: "2026-04-17T17:04:41Z" message: InferenceService [success-200-isvc-6941b] is Ready metadata: creationTimestamp: "2026-04-17T17:04:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:41Z" name: success-200-isvc-6941b.18a733c0fbd83dce namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: 0fa14d84-c623-49ab-b989-a64c8a192c28 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:04:29.063593Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7afa6-predictor-7549c68964-lvtm5 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:04:29Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733bdfa652a0f namespace: kserve-ci-e2e-test resourceVersion: "26177" uid: c33efb4f-f037-4d43-82f5-974a2a9af58b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7afa6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:29Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733be072ec966 namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: 884075cd-4930-470f-ac4d-a69c0ff41acf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26187" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:04:30Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733be37441a34 namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: f2a7ecc4-4483-402e-9da5-f9cf6540f98f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733be388d92e0 namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: b1083698-dee3-4e2f-87d0-5d5012b49086 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733be3bfd8a2e namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 10b7f73b-844b-4816-9942-924b112c2a35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733be3c6c1d85 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 11d06a78-9303-4c71-8165-044f73d42258 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733be3c7a090f namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: 383b5ce1-e995-4cb8-b64f-3ad016e7e16b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733be405af3e9 namespace: kserve-ci-e2e-test resourceVersion: "26250" uid: 615620fe-574d-4717-8933-7588f5652086 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:04:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:30Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733be40d180ec namespace: kserve-ci-e2e-test resourceVersion: "26252" uid: 9bd7ef17-e8be-4bb8-9d16-7a10db309415 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T17:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:05:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:05:17Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a733be6e9b9b8e namespace: kserve-ci-e2e-test resourceVersion: "26706" uid: 9112e1b2-d57f-4a78-82fd-baecac43233b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:13:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:13:53Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a734417f7964c6 namespace: kserve-ci-e2e-test resourceVersion: "30408" uid: 587d7644-43d2-42f8-911e-830adf01e76e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:13:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:13:53Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a734417f79c17b namespace: kserve-ci-e2e-test resourceVersion: "30409" uid: f0665518-8b8a-4953-b54e-c8bf61aa1f4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:13:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 0086f8ec-cd25-4666-a36c-d27f5ee1ca4f kind: Event lastTimestamp: "2026-04-17T17:13:58Z" message: 'Readiness probe failed: Get "https://10.132.0.48:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-04-17T17:13:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:13:58Z" name: success-200-isvc-7afa6-predictor-7549c68964-lvtm5.18a7344272b376c5 namespace: kserve-ci-e2e-test resourceVersion: "30459" uid: 74dd9f8f-4ee3-485e-b96f-8f21ccdd6165 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7afa6-predictor-7549c68964 namespace: kserve-ci-e2e-test resourceVersion: "26153" uid: c2965192-ce93-4351-9670-def7f18b55c2 kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: 'Created pod: success-200-isvc-7afa6-predictor-7549c68964-lvtm5' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:29Z" name: success-200-isvc-7afa6-predictor-7549c68964.18a733bdf9de20b6 namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: d0ddd531-4d4a-4346-bb23-036f89317ff2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26150" uid: 8665b5d1-1f8f-4edd-b4f6-83edb9c2f677 kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: Scaled up replica set success-200-isvc-7afa6-predictor-7549c68964 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:29Z" name: success-200-isvc-7afa6-predictor.18a733bdf7f16777 namespace: kserve-ci-e2e-test resourceVersion: "26156" uid: e42cf25c-174e-4205-bd11-20f7fcdaa66c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:04:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 50ade7b2-fa06-4955-94f1-4c423a47b2bf kind: Event lastTimestamp: "2026-04-17T17:04:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:59Z" name: success-200-isvc-7afa6-predictor.18a733c17661f1ca namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: bb87e7af-bc6b-4c4e-bfd8-1b2db219029d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:04:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 50ade7b2-fa06-4955-94f1-4c423a47b2bf kind: Event lastTimestamp: "2026-04-17T17:04:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:59Z" name: success-200-isvc-7afa6-predictor.18a733c176c3dffb namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: bacc01c5-661a-4bb4-94e8-3c395906b307 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 50ade7b2-fa06-4955-94f1-4c423a47b2bf kind: Event lastTimestamp: "2026-04-17T17:05:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:44Z" name: success-200-isvc-7afa6-predictor.18a733c8749460c6 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 079d9ef1-a372-4717-9a4c-bb54c0236d93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7afa6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 50ade7b2-fa06-4955-94f1-4c423a47b2bf kind: Event lastTimestamp: "2026-04-17T17:05:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:44Z" name: success-200-isvc-7afa6-predictor.18a733c874de3d04 namespace: kserve-ci-e2e-test resourceVersion: "26956" uid: 9ecdfc29-0824-4e7a-b665-a10ab3a78fa3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 53d05561-39e1-4f55-b381-c9c315924f00 kind: Event lastTimestamp: "2026-04-17T17:05:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:05:27Z" name: success-200-isvc-7afa6.18a733bdf7b6f422 namespace: kserve-ci-e2e-test resourceVersion: "26792" uid: 157f2209-f5e9-4ae1-9bf4-f756b4426ecb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 53d05561-39e1-4f55-b381-c9c315924f00 kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-7afa6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7afa6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:29Z" name: success-200-isvc-7afa6.18a733bdf8106c67 namespace: kserve-ci-e2e-test resourceVersion: "26159" uid: 6423db77-23e7-4520-a596-4b556afdf123 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 53d05561-39e1-4f55-b381-c9c315924f00 kind: Event lastTimestamp: "2026-04-17T17:04:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7afa6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:29Z" name: success-200-isvc-7afa6.18a733bdf810b365 namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: 648bb60d-3b41-41db-9894-fb4cecb86361 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7afa6 namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: 53d05561-39e1-4f55-b381-c9c315924f00 kind: Event lastTimestamp: "2026-04-17T17:05:27Z" message: InferenceService [success-200-isvc-7afa6] is Ready metadata: creationTimestamp: "2026-04-17T17:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:05:27Z" name: success-200-isvc-7afa6.18a733cb7d21a4b5 namespace: kserve-ci-e2e-test resourceVersion: "26791" uid: dd700234-4414-40c6-9750-e5d7f1c625a3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:43:54.378674Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8b822-predictor-74448fdcdb-6sqxm to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7329e816e995d namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 5916c0ef-96ff-45bd-8ad4-77ad840c2940 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15662" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7329e9abf47a1 namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: a57fa555-b180-4f36-84ce-4366b0726697 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7329e9c105b4c namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 019ebce0-fc5e-4943-aa12-7b2824f5ad05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7329e9eef8ff4 namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: dce9e149-fe54-4f28-a8f1-fa727399a898 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7329e9f62f372 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 3a2e16c1-f794-42c8-ada5-5f2e4fc7e57f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7329e9f6f2915 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 004c7332-68df-4894-a6a2-ce5071d287a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7329ea27d8e42 namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: b9970cd6-58be-4553-b7ec-36bc938700a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7329ea2edc4a6 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 80259fe0-a3b8-4f54-b1f9-70c62f412e8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:53:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:11Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7329ebbdd4482 namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: 746a1b9f-10ea-4ad1-9bc4-22822d1ba82a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7331fab2213b4 namespace: kserve-ci-e2e-test resourceVersion: "19956" uid: 00b78bd5-420a-48b3-a8cd-0a52949c6c60 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7331fab26aa48 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 8139f013-027f-4bfd-95ab-d8cf8310d492 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 432dc8ff-e1ad-414b-a836-0f16a600f03e kind: Event lastTimestamp: "2026-04-17T16:53:11Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:11Z" name: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm.18a7332030733a39 namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 521cbe10-8d37-4483-bc0c-5292338233e3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8b822-predictor-74448fdcdb namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: cc24d50f-5bcb-4e63-afaf-c46c18843365 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: 'Created pod: success-200-isvc-8b822-predictor-74448fdcdb-6sqxm' metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor-74448fdcdb.18a7329e810044f2 namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: 74d748e1-24a1-4539-b2c9-43f7085b24b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 2c4fc7ea-70fd-4b61-9aea-86e1799819d3 kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: Scaled up replica set success-200-isvc-8b822-predictor-74448fdcdb from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822-predictor.18a7329e7f3db1bb namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: b29d5964-358b-4c5a-9018-30a8000dc7fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:44:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 13e96031-c187-4dbe-8e9d-4b450da8adca kind: Event lastTimestamp: "2026-04-17T16:44:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:39Z" name: success-200-isvc-8b822-predictor.18a732a1fdcc2ad3 namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: bb75c928-5fda-449c-a26d-6a99af788745 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:44:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 13e96031-c187-4dbe-8e9d-4b450da8adca kind: Event lastTimestamp: "2026-04-17T16:44:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:44:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:39Z" name: success-200-isvc-8b822-predictor.18a732a1fe31c874 namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 93104b0a-cf3e-422b-8fc9-1d23416eee16 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 13e96031-c187-4dbe-8e9d-4b450da8adca kind: Event lastTimestamp: "2026-04-17T16:45:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:09Z" name: success-200-isvc-8b822-predictor.18a732ac7a89e38a namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 634121ee-1ac4-423c-b8c3-d2c3de650de3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b822-predictor namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 13e96031-c187-4dbe-8e9d-4b450da8adca kind: Event lastTimestamp: "2026-04-17T16:45:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:09Z" name: success-200-isvc-8b822-predictor.18a732ac7b041335 namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: dfb8b309-31e4-4c1f-bb4d-453cb7477599 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b822 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 285b2635-b07b-4471-8fe3-6b73436f57ce kind: Event lastTimestamp: "2026-04-17T16:44:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:41Z" name: success-200-isvc-8b822.18a7329e7f3bf130 namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: f3fcc372-834b-447e-a424-acbdc29814cc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b822 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 285b2635-b07b-4471-8fe3-6b73436f57ce kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: 'Failed to update status for InferenceService "success-200-isvc-8b822": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8b822": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822.18a7329e7fe8a6a0 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 75910579-5030-4435-8a9e-0596100eeb7d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b822 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 285b2635-b07b-4471-8fe3-6b73436f57ce kind: Event lastTimestamp: "2026-04-17T16:43:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8b822": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:54Z" name: success-200-isvc-8b822.18a7329e7fe8dd9e namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 092945b9-8756-410e-8031-9b4dc7aaddf0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b822 namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: 285b2635-b07b-4471-8fe3-6b73436f57ce kind: Event lastTimestamp: "2026-04-17T16:44:41Z" message: InferenceService [success-200-isvc-8b822] is Ready metadata: creationTimestamp: "2026-04-17T16:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:41Z" name: success-200-isvc-8b822.18a732a976f9f8d6 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 251e0f2e-8563-4d0c-9465-99158a533d42 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:41:44.196165Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14035" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9e0ff-predictor-598855d998-zbbsz to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:41:44Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a7328031f3346b namespace: kserve-ci-e2e-test resourceVersion: "14041" uid: c298702b-2982-4fd6-8fa0-2899d2bb51f5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14044" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:41:44Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a732804ba87281 namespace: kserve-ci-e2e-test resourceVersion: "14095" uid: e41ea6e3-c25b-4105-886f-031b4bb3da55 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:44Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a732804cfc1c5d namespace: kserve-ci-e2e-test resourceVersion: "14099" uid: f2f2226e-6c60-4b14-be49-e941e896a34e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:41:59Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" in 14.797s (14.797s including waiting). Image size: 1332303459 bytes.' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:59Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a73283befbf7f4 namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 342e3415-56f3-422e-ba45-c6e9442ca685 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:41:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:59Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a73283c33422f1 namespace: kserve-ci-e2e-test resourceVersion: "14320" uid: 8f258b69-27ed-4cec-b527-9995ac214d31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:41:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:59Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a73283c430e5df namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: 1f93557d-3f9b-4c89-94cb-dbb8292b351f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:41:59Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:59Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a73283c43f658b namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 87dde283-8d94-466a-ba10-a0d3d7b43357 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:42:03Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.993s (3.993s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-17T16:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:03Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a73284b24b9433 namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: dadc0960-77da-4262-9794-d13f1b8c4a7b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:42:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:03Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a73284b6e33729 namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: 4153343a-f6c7-45b8-b876-aa41a3f8474f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:42:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:03Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a73284b76a64d3 namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 33908fd6-8963-4d01-b50e-bf12ab85440b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:43:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:21Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a732850a9ea6ef namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: 3a5d1d4d-3469-4ca2-8543-263e736e60a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:18Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a732962281fae5 namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: 9fb10ed9-19d2-45be-ad14-18b3a5e7b468 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:18Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a732962282d463 namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 94d790ab-a441-43f1-a984-ac2bd9a67e12 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 52b1d4cc-638e-41b1-9fdb-63687ef13969 kind: Event lastTimestamp: "2026-04-17T16:43:21Z" message: 'Readiness probe failed: Get "https://10.132.0.20:8643/healthz": dial tcp 10.132.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:21Z" name: success-200-isvc-9e0ff-predictor-598855d998-zbbsz.18a73296bcbda3b5 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 3518b22a-80e7-4c54-bc13-ddb9272c20ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9e0ff-predictor-598855d998 namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 4ef452b8-e26c-4ae3-8b82-b381b30c1fe8 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: 'Created pod: success-200-isvc-9e0ff-predictor-598855d998-zbbsz' metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:44Z" name: success-200-isvc-9e0ff-predictor-598855d998.18a73280317bbb29 namespace: kserve-ci-e2e-test resourceVersion: "14037" uid: 0f7e7adb-bf96-4db1-b351-2606163db6c0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 9bc7da1a-af69-4bbc-80d6-8279bef735ae kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Scaled up replica set success-200-isvc-9e0ff-predictor-598855d998 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:44Z" name: success-200-isvc-9e0ff-predictor.18a732802fc92c32 namespace: kserve-ci-e2e-test resourceVersion: "14032" uid: d3af8ab3-753c-489f-8711-226727b755a4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14028" uid: 5ab3209a-59b2-47e8-a85c-bdd27c449362 kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: success-200-isvc-9e0ff-predictor.18a73283ae40b708 namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: ab8096a4-beda-4951-9fba-3942f9865bac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14028" uid: 5ab3209a-59b2-47e8-a85c-bdd27c449362 kind: Event lastTimestamp: "2026-04-17T16:42:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:29Z" name: success-200-isvc-9e0ff-predictor.18a73283aea03993 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: e08265e3-bdeb-4e97-835c-c15087ddbf20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 5ab3209a-59b2-47e8-a85c-bdd27c449362 kind: Event lastTimestamp: "2026-04-17T16:43:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:14Z" name: success-200-isvc-9e0ff-predictor.18a7328e2aab5ebf namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: c500b153-4f13-45a7-bf94-e69288b4d228 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9e0ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 5ab3209a-59b2-47e8-a85c-bdd27c449362 kind: Event lastTimestamp: "2026-04-17T16:43:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:14Z" name: success-200-isvc-9e0ff-predictor.18a7328e2b09a20d namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 0f611792-f276-4470-87e0-3619fe5ed3f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9e0ff namespace: kserve-ci-e2e-test resourceVersion: "14019" uid: 6b42fd11-ee7e-44bb-8c05-a4c2f09ffae5 kind: Event lastTimestamp: "2026-04-17T16:42:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:42:51Z" name: success-200-isvc-9e0ff.18a732802fa401e4 namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 729f2f3e-6f6e-4ee3-b7cb-d9cdb2048888 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9e0ff namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 6b42fd11-ee7e-44bb-8c05-a4c2f09ffae5 kind: Event lastTimestamp: "2026-04-17T16:42:51Z" message: InferenceService [success-200-isvc-9e0ff] is Ready metadata: creationTimestamp: "2026-04-17T16:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:42:51Z" name: success-200-isvc-9e0ff.18a7328fc5daeb93 namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: c2e7c290-4eaf-4e05-9f48-47fa3ce5d450 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:52:33.570480Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19553" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ac602-predictor-757bd75446-krh9b to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:52:33Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a7331763acd173 namespace: kserve-ci-e2e-test resourceVersion: "19563" uid: b1763b69-0c96-4a7c-8d16-957f73fa9737 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ac602-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a733176cecfa5a namespace: kserve-ci-e2e-test resourceVersion: "19591" uid: 1e9c4269-3e60-4fd1-8281-a1f84605f174 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19574" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:52:34Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a73317a0df023e namespace: kserve-ci-e2e-test resourceVersion: "19610" uid: 2b87e118-6a48-4458-b595-b06d1b2028ee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a73317a2532fd8 namespace: kserve-ci-e2e-test resourceVersion: "19613" uid: dfbff361-c8c2-4c68-8f6b-357ab29267e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a73317a61e9434 namespace: kserve-ci-e2e-test resourceVersion: "19616" uid: 2577d412-a1e0-4c58-8439-0daf5d3230b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a73317a6971bc5 namespace: kserve-ci-e2e-test resourceVersion: "19618" uid: e7a6ebf4-e7e2-47fe-8328-9c8a8b6db3e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a73317a6a2b626 namespace: kserve-ci-e2e-test resourceVersion: "19619" uid: 6e3d3672-25ef-4e32-a1ae-02928e360856 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a73317a9bd5f6b namespace: kserve-ci-e2e-test resourceVersion: "19620" uid: 0c022a64-ce19-48e7-8c62-49d595e1dca3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:52:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:34Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a73317aa31aa1f namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: 37dd9405-7604-402f-a6ac-6792db1fc418 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:52:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:53:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:52:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:21Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a73317eb4312d6 namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: 412f7d23-5fd9-47cc-83ca-f11f4c67f7c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:53Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a7332a0b418c35 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: d77a030e-220f-4688-96aa-194d46d6ac54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:53Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a7332a0b421802 namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: f16e5686-5c09-48f8-9831-4f4bf435d416 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac602-predictor-757bd75446-krh9b namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 8eef08cd-81aa-4be2-9abc-b72a91020ae2 kind: Event lastTimestamp: "2026-04-17T16:53:56Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:56Z" name: success-200-isvc-ac602-predictor-757bd75446-krh9b.18a7332ac786c437 namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 6bff22d0-9487-473b-a9ba-30948585bcf7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ac602-predictor-757bd75446 namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: cec8b701-2845-409f-b0f6-c3805da6850f kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'Created pod: success-200-isvc-ac602-predictor-757bd75446-krh9b' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:33Z" name: success-200-isvc-ac602-predictor-757bd75446.18a733176310f6a3 namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: 77ae599d-597b-47fa-a020-71c2e7e230c0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19526" uid: 3cc5cb5c-e743-4d9a-b8a4-a45094d61089 kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Scaled up replica set success-200-isvc-ac602-predictor-757bd75446 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:33Z" name: success-200-isvc-ac602-predictor.18a733175cec2dbe namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: fe781a17-04c1-43d9-b818-940d1e698c25 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:52:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: 6dcbb522-cb5f-48df-968a-31ff1a81906f kind: Event lastTimestamp: "2026-04-17T16:53:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:03Z" name: success-200-isvc-ac602-predictor.18a7331adbd03d9a namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: 9209403b-bdab-45bc-a42a-4b4d5f351d79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:52:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: 6dcbb522-cb5f-48df-968a-31ff1a81906f kind: Event lastTimestamp: "2026-04-17T16:53:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:03Z" name: success-200-isvc-ac602-predictor.18a7331adc322a88 namespace: kserve-ci-e2e-test resourceVersion: "19868" uid: 0e005004-5344-44a7-8f91-93b14cb25529 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19755" uid: 6dcbb522-cb5f-48df-968a-31ff1a81906f kind: Event lastTimestamp: "2026-04-17T16:53:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:48Z" name: success-200-isvc-ac602-predictor.18a73321da07b8cb namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 323c3e75-516f-4a14-a115-bd072b45428e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac602-predictor namespace: kserve-ci-e2e-test resourceVersion: "19755" uid: 6dcbb522-cb5f-48df-968a-31ff1a81906f kind: Event lastTimestamp: "2026-04-17T16:53:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:48Z" name: success-200-isvc-ac602-predictor.18a73321da72d4ca namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: 4a604210-9ed3-4b07-ac49-1665446ed4c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac602 namespace: kserve-ci-e2e-test resourceVersion: "19502" uid: 648c3b91-b855-48d9-98d6-b20b8cf39dca kind: Event lastTimestamp: "2026-04-17T16:53:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:31Z" name: success-200-isvc-ac602.18a733175cd4c00f namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: 0ab25924-d33c-4de1-a526-dfb5c72477dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac602 namespace: kserve-ci-e2e-test resourceVersion: "19502" uid: 648c3b91-b855-48d9-98d6-b20b8cf39dca kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'Failed to update status for InferenceService "success-200-isvc-ac602": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ac602": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:52:33Z" name: success-200-isvc-ac602.18a733175deb97b1 namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: c2330f97-2a65-4796-86f1-ab9980838b61 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac602 namespace: kserve-ci-e2e-test resourceVersion: "19502" uid: 648c3b91-b855-48d9-98d6-b20b8cf39dca kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ac602": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:52:33Z" name: success-200-isvc-ac602.18a733175debee30 namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: c95b2665-5077-48b3-9001-cf1997e11102 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac602 namespace: kserve-ci-e2e-test resourceVersion: "20260" uid: 648c3b91-b855-48d9-98d6-b20b8cf39dca kind: Event lastTimestamp: "2026-04-17T16:53:31Z" message: InferenceService [success-200-isvc-ac602] is Ready metadata: creationTimestamp: "2026-04-17T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:31Z" name: success-200-isvc-ac602.18a73324f9dcae3d namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: efa1752c-c66a-44fa-a4d4-4e5c5ff899cc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:53:53.724299Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:53:53Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a7332a0d371423 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: a4cfb8dd-7817-4c8e-9c3a-ca66956a2359 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:53:54Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a7332a271fc1c6 namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: 6690d880-6a2c-48d7-afc4-f387603f7ae0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a7332a2882171e namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: dd04df9b-d7f4-404c-860e-70be4ce7df4c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a7332a2c748467 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 1331e2df-519a-4c98-a98a-7f3ce3516d8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a7332a2cfaeb9b namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 593006ab-068e-45ff-a271-9b127c544d91 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a7332a2d0a58cc namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 4fa1efd1-ea99-47f9-850b-76aa4def9f73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a7332a3173d27d namespace: kserve-ci-e2e-test resourceVersion: "20602" uid: 8c4ce3db-5a36-4e11-bdf2-ebbe2480a532 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T16:53:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:54Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a7332a31ede3ea namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: 2105402a-d1c0-4b4d-871b-3b246c1bc60f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T16:54:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:42Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a7332a9c79c8ca namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 0309c454-eb08-4b35-bcc6-9940d7bb83ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:18Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a733ad9bda487f namespace: kserve-ci-e2e-test resourceVersion: "24966" uid: e95ef7ed-25c3-4257-9cd7-42eb4241c5a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T17:03:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:18Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a733ad9bdac231 namespace: kserve-ci-e2e-test resourceVersion: "24968" uid: 214ff144-8251-4495-b7a4-92f5635a56f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T17:03:23Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-04-17T17:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:23Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a733aea065eaf0 namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 4d67524e-451a-427f-8a27-edf7165bc9fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: b10bfd64-2fd9-43bd-9507-303d487de5a1 kind: Event lastTimestamp: "2026-04-17T17:03:23Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: i/o timeout' metadata: creationTimestamp: "2026-04-17T17:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:23Z" name: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6.18a733aea0c3a0f2 namespace: kserve-ci-e2e-test resourceVersion: "25093" uid: 1e3de7b2-42d4-4b8c-adbd-ccd7d7d6181c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c68dd-predictor-57f664bd48 namespace: kserve-ci-e2e-test resourceVersion: "20518" uid: e113803f-3428-4723-a3e5-a10078eb4f8d kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: 'Created pod: success-200-isvc-c68dd-predictor-57f664bd48-xzcc6' metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:53Z" name: success-200-isvc-c68dd-predictor-57f664bd48.18a7332a0caabec2 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 7d7d8b83-e746-4b9b-92e2-1cf8a0e2d96d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 80b8a043-1464-4a64-afb9-9d9a1ce2973d kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: Scaled up replica set success-200-isvc-c68dd-predictor-57f664bd48 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:53Z" name: success-200-isvc-c68dd-predictor.18a7332a0ae3fa1e namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 83815c3b-1827-4d20-bc31-94bea63b4a71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: bd473e7f-069c-4cb1-9a4c-5927eed3abc9 kind: Event lastTimestamp: "2026-04-17T16:54:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:38Z" name: success-200-isvc-c68dd-predictor.18a7332d89529cd3 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: e78a95b4-4df5-4320-a859-0ff1a7ee27f8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: bd473e7f-069c-4cb1-9a4c-5927eed3abc9 kind: Event lastTimestamp: "2026-04-17T16:54:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:38Z" name: success-200-isvc-c68dd-predictor.18a7332d89ab0a9a namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 28be1508-db58-47d3-90db-98de9be0f40d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:54:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: bd473e7f-069c-4cb1-9a4c-5927eed3abc9 kind: Event lastTimestamp: "2026-04-17T16:55:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:23Z" name: success-200-isvc-c68dd-predictor.18a7333807080137 namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 6c3b9796-d274-4835-862c-5e6cd988b9ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:54:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c68dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: bd473e7f-069c-4cb1-9a4c-5927eed3abc9 kind: Event lastTimestamp: "2026-04-17T16:55:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:23Z" name: success-200-isvc-c68dd-predictor.18a7333807734dcd namespace: kserve-ci-e2e-test resourceVersion: "21614" uid: b33457af-3094-4cfb-90dc-cca6b4e3bf70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c68dd namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: d553a2f9-0351-4431-abe7-70f0cb6d2568 kind: Event lastTimestamp: "2026-04-17T16:54:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:52Z" name: success-200-isvc-c68dd.18a7332a0aace358 namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: 5e6a39b9-4c33-403f-bb29-18e5cb9d489a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c68dd namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: d553a2f9-0351-4431-abe7-70f0cb6d2568 kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: 'Failed to update status for InferenceService "success-200-isvc-c68dd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c68dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:53Z" name: success-200-isvc-c68dd.18a7332a0b49a337 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: f921fbdd-58ca-45ea-8224-ddbc7f7d670a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c68dd namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: d553a2f9-0351-4431-abe7-70f0cb6d2568 kind: Event lastTimestamp: "2026-04-17T16:53:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c68dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:53Z" name: success-200-isvc-c68dd.18a7332a0b4a0791 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: e270796c-5817-48a7-9a1c-096fd3a7dab7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c68dd namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: d553a2f9-0351-4431-abe7-70f0cb6d2568 kind: Event lastTimestamp: "2026-04-17T16:54:52Z" message: InferenceService [success-200-isvc-c68dd] is Ready metadata: creationTimestamp: "2026-04-17T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:52Z" name: success-200-isvc-c68dd.18a73337ab3c2061 namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 5bc53ee8-fd9f-4fc5-a34c-41f1e1e2468c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:43:18.504561Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ce4d4-predictor-5d646b757d-msdzr to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:43:18Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a73296272b259b namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 0c8e6c33-718d-4a78-81ec-fd53cd8a995d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:43:18Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a73296411b51d2 namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: b82e8454-b244-4e9b-b48d-1d3d6b33e4c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:18Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a73296428e36a7 namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: f6dcee01-64f4-47d2-a90a-abe69fbeff25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a732964604b973 namespace: kserve-ci-e2e-test resourceVersion: "15203" uid: 3b461bc7-e29d-4981-ac3f-b703d86cad3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a73296468ebbbd namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: 68ae0657-c07b-4e18-b537-7495aa63006f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a73296469b8754 namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 592251cf-69d2-47f4-9eeb-59e1e56a3be8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a7329649d2fffe namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 121f407d-d48f-4a31-95d8-bdd3f2a305c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:43:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:19Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a732964a466332 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 665ee438-52e5-4d18-94c9-368f3c2d6bf5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:52:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:36Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a7329652b4b76a namespace: kserve-ci-e2e-test resourceVersion: "19652" uid: fa8aa41e-f6c5-41fe-8f8b-f7d18b07f925 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a7331759d17b99 namespace: kserve-ci-e2e-test resourceVersion: "19524" uid: cd0984dc-7244-46bf-bc44-c59b5d19a0da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a7331759d2342a namespace: kserve-ci-e2e-test resourceVersion: "19529" uid: 8423742e-05a8-441f-9d2b-55a71667ad2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: b08b81ef-3e96-4cd5-ac27-13a36d7be0c6 kind: Event lastTimestamp: "2026-04-17T16:52:36Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:52:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:36Z" name: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr.18a7331802fa5949 namespace: kserve-ci-e2e-test resourceVersion: "19649" uid: 4a2b685a-5341-476e-8bde-65795a4d200d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ce4d4-predictor-5d646b757d namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: ca1d4b28-0bb5-4790-9122-1c20b510ded0 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: 'Created pod: success-200-isvc-ce4d4-predictor-5d646b757d-msdzr' metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:18Z" name: success-200-isvc-ce4d4-predictor-5d646b757d.18a7329626c7c08f namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: 5509605c-1fc2-4a60-867b-167f55a25e2c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: dfc4463f-c172-4cff-be5a-8d9bcf72c9e8 kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: Scaled up replica set success-200-isvc-ce4d4-predictor-5d646b757d from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:18Z" name: success-200-isvc-ce4d4-predictor.18a732962348b5da namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: 502b9e6e-a410-4893-8950-82a2231723ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: a0db4c19-1355-4f86-bd91-78a854e30dca kind: Event lastTimestamp: "2026-04-17T16:44:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:33Z" name: success-200-isvc-ce4d4-predictor.18a73299a1d5b6e1 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: f64848f5-4579-4cd0-8810-bbdfdfe6a23b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: a0db4c19-1355-4f86-bd91-78a854e30dca kind: Event lastTimestamp: "2026-04-17T16:44:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:33Z" name: success-200-isvc-ce4d4-predictor.18a73299a236745b namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: 3d218ce6-554e-446b-8f3d-ec57b7e98665 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:44:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: a0db4c19-1355-4f86-bd91-78a854e30dca kind: Event lastTimestamp: "2026-04-17T16:44:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:18Z" name: success-200-isvc-ce4d4-predictor.18a732a0a03b5b29 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 7c1cf318-d82a-4b6c-b968-517459257eb9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:44:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce4d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: a0db4c19-1355-4f86-bd91-78a854e30dca kind: Event lastTimestamp: "2026-04-17T16:44:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:18Z" name: success-200-isvc-ce4d4-predictor.18a732a0a0a0b0f4 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 1829c551-3a3d-4af5-961e-da5f0472d924 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 0f66ea35-c9bd-4ad4-96fa-0efd0df2fb0e kind: Event lastTimestamp: "2026-04-17T16:44:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:06Z" name: success-200-isvc-ce4d4.18a73296231a1fa4 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: 7f20a249-fff8-43af-9d3a-75998bfddfcc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 0f66ea35-c9bd-4ad4-96fa-0efd0df2fb0e kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: 'Failed to update status for InferenceService "success-200-isvc-ce4d4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ce4d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:18Z" name: success-200-isvc-ce4d4.18a732962386e556 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: 23a2b5b7-533a-4915-9357-eb5b66b4f3d7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 0f66ea35-c9bd-4ad4-96fa-0efd0df2fb0e kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ce4d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:18Z" name: success-200-isvc-ce4d4.18a7329623872ac5 namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 9e61f55c-eaba-4c3e-ab0f-61c3029d4bdd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 0f66ea35-c9bd-4ad4-96fa-0efd0df2fb0e kind: Event lastTimestamp: "2026-04-17T16:44:06Z" message: InferenceService [success-200-isvc-ce4d4] is Ready metadata: creationTimestamp: "2026-04-17T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:06Z" name: success-200-isvc-ce4d4.18a732a14c28fb8c namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: d6d556f9-d9d1-41ab-9cc2-f5590ae52967 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:53:09.181069Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19972" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d0d22-predictor-65f777864b-fb2rr to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a7331fae3b6120 namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: da70a78c-fe50-4b1f-b569-b92275d52932 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a7331fc76aebb5 namespace: kserve-ci-e2e-test resourceVersion: "20028" uid: faacb213-c39e-4899-9f9f-dedef7838ac5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a7331fc8d40295 namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 473adde2-38ec-4a6c-ae24-0b0b47b41063 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a7331fcbd065d6 namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 9f5c7f89-9a8d-4e8a-9586-e6fe50e137e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a7331fcc414596 namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 8644ea34-33d9-4033-b0b6-87efa6b2a396 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a7331fcc4d1294 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: d64b2964-5532-4d76-9ddb-43924ea7dbd9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a7331fcf660334 namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: b7465b6e-d60d-4e73-9f9e-8080081cae27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a7331fcfd911a0 namespace: kserve-ci-e2e-test resourceVersion: "20036" uid: dfee374c-0fea-4fbd-afb5-4966780d7cb9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:53:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:53:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:56Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a7332018a8c4d6 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 394507b6-e31c-4939-bfdc-dab756fd26c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:29Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a733325bc0d7ff namespace: kserve-ci-e2e-test resourceVersion: "21016" uid: 2f23b48e-264d-4728-8330-79a5d15167fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:29Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a733325bc15b94 namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: f67887ea-c285-41b0-909d-054d806e462a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: a6f8ebf7-b46f-4015-b8d2-49f38ee1dceb kind: Event lastTimestamp: "2026-04-17T16:54:31Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:31Z" name: success-200-isvc-d0d22-predictor-65f777864b-fb2rr.18a73332f4c63693 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: cd9c6f5a-0344-4f72-8af9-71771da081fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d0d22-predictor-65f777864b namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 961501d5-0bea-4250-abf1-4d3bf58389ba kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: 'Created pod: success-200-isvc-d0d22-predictor-65f777864b-fb2rr' metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor-65f777864b.18a7331fadc9fdc6 namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 7259c68f-f0a3-46b8-9da4-bb6de3e54566 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "19955" uid: 4d567978-0cf5-434a-afc9-5e27809bf1d3 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: Scaled up replica set success-200-isvc-d0d22-predictor-65f777864b from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22-predictor.18a7331fac116d03 namespace: kserve-ci-e2e-test resourceVersion: "19961" uid: f7fed071-b86c-4f04-96ad-910c58a2f276 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:53:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "19957" uid: 406a8f91-9c33-47ed-bdc5-10b56c6f0b37 kind: Event lastTimestamp: "2026-04-17T16:54:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:24Z" name: success-200-isvc-d0d22-predictor.18a733232ab61a29 namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: 8b2a1965-bf60-4e78-bc23-cb5fff2795e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:53:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "19957" uid: 406a8f91-9c33-47ed-bdc5-10b56c6f0b37 kind: Event lastTimestamp: "2026-04-17T16:54:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:53:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:24Z" name: success-200-isvc-d0d22-predictor.18a733232b1f6f67 namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: 8057936f-a20c-401a-bf57-6e276dbf3828 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 406a8f91-9c33-47ed-bdc5-10b56c6f0b37 kind: Event lastTimestamp: "2026-04-17T16:54:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:09Z" name: success-200-isvc-d0d22-predictor.18a73326a9c58b27 namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: ad4c7ff0-3dcf-4ee4-bf5d-acc49b332da7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0d22-predictor namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 406a8f91-9c33-47ed-bdc5-10b56c6f0b37 kind: Event lastTimestamp: "2026-04-17T16:54:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:09Z" name: success-200-isvc-d0d22-predictor.18a73326aa23c60c namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 16223e24-eb80-4149-84b9-76e7a7cc512a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "19933" uid: fa6389f2-1e08-4854-a873-3b0295b19cb7 kind: Event lastTimestamp: "2026-04-17T16:54:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:07Z" name: success-200-isvc-d0d22.18a7331fabe8e897 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: fc831096-762b-4c39-83ea-359d816d0fd7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "19933" uid: fa6389f2-1e08-4854-a873-3b0295b19cb7 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: 'Failed to update status for InferenceService "success-200-isvc-d0d22": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d0d22": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22.18a7331fac508b2d namespace: kserve-ci-e2e-test resourceVersion: "19963" uid: 461b30a2-7f98-4688-82da-ee9c14c17459 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "19933" uid: fa6389f2-1e08-4854-a873-3b0295b19cb7 kind: Event lastTimestamp: "2026-04-17T16:53:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d0d22": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:09Z" name: success-200-isvc-d0d22.18a7331fac50d0ca namespace: kserve-ci-e2e-test resourceVersion: "19965" uid: 0ff05ffa-25eb-481a-9bbb-01df010a7e8a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0d22 namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: fa6389f2-1e08-4854-a873-3b0295b19cb7 kind: Event lastTimestamp: "2026-04-17T16:54:07Z" message: InferenceService [success-200-isvc-d0d22] is Ready metadata: creationTimestamp: "2026-04-17T16:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:07Z" name: success-200-isvc-d0d22.18a7332d27edf969 namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: 2c695bc9-f83a-4582-9400-c5eb8769ae46 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:54:29.429497Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f6018-predictor-55f969cd7b-vv7hw to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:54:29Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a733325d692081 namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: bee7372d-4ccf-4197-9368-d4e8950b219d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f6018-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:29Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a733326804ccf1 namespace: kserve-ci-e2e-test resourceVersion: "21072" uid: 93e732b9-507f-406f-8322-fa44b4497521 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:54:30Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a733329ab02fbe namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 41e3c812-7fd3-40ff-b957-62dc9d76f6a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a733329bf144c1 namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 79aafb12-e8fd-4052-a18e-a365cfe1f56b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a733329ed3356d namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: ac3bcfb1-2437-44c5-9e6c-aceec595dc94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a733329f503e1e namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 29aa99ba-3087-4175-9502-fc0496439017 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a733329f5bcdfc namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 4279ec4f-3afa-4239-b5ae-cc8540345d6d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a73332a21a001f namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 2abe0b2c-a8c6-4241-9f98-9c9caa7c943f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T16:54:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:54:30Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a73332a28cc437 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 797d92c7-aa44-49c5-8c51-90cc470673c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T16:55:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:55:08Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a7333305a0c031 namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: aa29d66a-023e-45db-965c-5fffe145aa21 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:44Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a733b38f69a92b namespace: kserve-ci-e2e-test resourceVersion: "25481" uid: c156d586-8e76-4689-97ff-18a5dcd81bec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:03:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 6b7f5f6d-d978-4a83-8d99-40e024607e2b kind: Event lastTimestamp: "2026-04-17T17:03:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:03:44Z" name: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw.18a733b38f6a6027 namespace: kserve-ci-e2e-test resourceVersion: "25483" uid: 027fa32c-f1d0-4d73-8152-e1e12b77a9aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f6018-predictor-55f969cd7b namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 8ddd4281-341d-4e39-9adc-7db4f7cadf99 kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: 'Created pod: success-200-isvc-f6018-predictor-55f969cd7b-vv7hw' metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:29Z" name: success-200-isvc-f6018-predictor-55f969cd7b.18a733325ccc90a0 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: f584af2e-fb89-4101-ad6f-35bd527143a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: fbbdac49-2168-4fa5-a0d6-fdbded39cfd9 kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: Scaled up replica set success-200-isvc-f6018-predictor-55f969cd7b from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:29Z" name: success-200-isvc-f6018-predictor.18a733325ae801fc namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: aff1d4f0-903c-4988-aec9-f42678081e43 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 76c0f351-c7b0-492e-9445-58171cdd61db kind: Event lastTimestamp: "2026-04-17T16:54:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:59Z" name: success-200-isvc-f6018-predictor.18a73335d989b030 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 6edbe624-9b39-4f83-a880-56e9dc79c2b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 76c0f351-c7b0-492e-9445-58171cdd61db kind: Event lastTimestamp: "2026-04-17T16:54:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:54:59Z" name: success-200-isvc-f6018-predictor.18a73335d9e97d91 namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 3a620453-2cf4-4357-a37b-aa9d6e6c8e12 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: 76c0f351-c7b0-492e-9445-58171cdd61db kind: Event lastTimestamp: "2026-04-17T16:55:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:29Z" name: success-200-isvc-f6018-predictor.18a7333cd757e265 namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: 451d2b89-57ef-44cf-8da1-bf4e17fafc66 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6018-predictor namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: 76c0f351-c7b0-492e-9445-58171cdd61db kind: Event lastTimestamp: "2026-04-17T16:55:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:55:29Z" name: success-200-isvc-f6018-predictor.18a7333cd7bcc6ef namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: ecdbc6bc-3ca4-4eef-9aa7-bd5c0b29ffbc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6018 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 8a7712db-77f0-4ff2-8e70-b72530b624dc kind: Event lastTimestamp: "2026-04-17T16:55:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:55:18Z" name: success-200-isvc-f6018.18a733325af2afd6 namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: b9f471aa-7f59-4c26-94c3-60e91ee582a0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6018 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 8a7712db-77f0-4ff2-8e70-b72530b624dc kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-f6018": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f6018": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:29Z" name: success-200-isvc-f6018.18a733325bc7a3a8 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 2e8b0eb1-7806-47e3-ae2b-4066d161a1e4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6018 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 8a7712db-77f0-4ff2-8e70-b72530b624dc kind: Event lastTimestamp: "2026-04-17T16:54:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f6018": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:54:29Z" name: success-200-isvc-f6018.18a733325bc7db58 namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: e8836a98-bd6e-4cec-ae6c-29c2b859b611 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:55:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6018 namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: 8a7712db-77f0-4ff2-8e70-b72530b624dc kind: Event lastTimestamp: "2026-04-17T16:55:18Z" message: InferenceService [success-200-isvc-f6018] is Ready metadata: creationTimestamp: "2026-04-17T16:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:55:18Z" name: success-200-isvc-f6018.18a7333dc0e09b81 namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: 7ea6a072-2000-4073-aa88-e39328a6b91a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:04:54.414148Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6941b-869c44b684-24nwx namespace: kserve-ci-e2e-test resourceVersion: "26493" uid: c9efa573-7e51-407e-9a19-731c1efdda17 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-6941b-869c44b684-24nwx to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T17:04:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:04:54Z" name: switch-graph-6941b-869c44b684-24nwx.18a733c3e1661676 namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: e65bc760-56db-42d7-b40a-ec655fa4853c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:54Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6941b-869c44b684-24nwx namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: c9efa573-7e51-407e-9a19-731c1efdda17 kind: Event lastTimestamp: "2026-04-17T17:04:54Z" message: Add eth0 [10.132.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:04:54Z" name: switch-graph-6941b-869c44b684-24nwx.18a733c3fb338180 namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: 88ffc63c-423a-4f10-bc09-33f5abb79856 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6941b} kind: Pod name: switch-graph-6941b-869c44b684-24nwx namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: c9efa573-7e51-407e-9a19-731c1efdda17 kind: Event lastTimestamp: "2026-04-17T17:04:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T17:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:54Z" name: switch-graph-6941b-869c44b684-24nwx.18a733c3fc683a60 namespace: kserve-ci-e2e-test resourceVersion: "26514" uid: 739f1c9b-d771-491b-8f42-dcb5145b3d32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6941b} kind: Pod name: switch-graph-6941b-869c44b684-24nwx namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: c9efa573-7e51-407e-9a19-731c1efdda17 kind: Event lastTimestamp: "2026-04-17T17:04:54Z" message: 'Created container: switch-graph-6941b' metadata: creationTimestamp: "2026-04-17T17:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:54Z" name: switch-graph-6941b-869c44b684-24nwx.18a733c3ffc9cc46 namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 1e3cd257-dc24-494e-9fff-8a2829120e3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6941b} kind: Pod name: switch-graph-6941b-869c44b684-24nwx namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: c9efa573-7e51-407e-9a19-731c1efdda17 kind: Event lastTimestamp: "2026-04-17T17:04:54Z" message: Started container switch-graph-6941b metadata: creationTimestamp: "2026-04-17T17:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:04:54Z" name: switch-graph-6941b-869c44b684-24nwx.18a733c4004902e2 namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 2085230e-86bc-4eaf-b851-744c4c2000ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:21:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6941b} kind: Pod name: switch-graph-6941b-869c44b684-24nwx namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: c9efa573-7e51-407e-9a19-731c1efdda17 kind: Event lastTimestamp: "2026-04-17T17:21:13Z" message: Stopping container switch-graph-6941b metadata: creationTimestamp: "2026-04-17T17:21:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:21:13Z" name: switch-graph-6941b-869c44b684-24nwx.18a734a7eb7fc316 namespace: kserve-ci-e2e-test resourceVersion: "33463" uid: 0dd3562c-fc50-40fe-80ff-f82ae3923ced reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:21:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6941b} kind: Pod name: switch-graph-6941b-869c44b684-24nwx namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: c9efa573-7e51-407e-9a19-731c1efdda17 kind: Event lastTimestamp: "2026-04-17T17:21:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:21:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:21:36Z" name: switch-graph-6941b-869c44b684-24nwx.18a734a872c22c14 namespace: kserve-ci-e2e-test resourceVersion: "33691" uid: 4c2529a6-a5ab-4eb7-ae88-c50d10d09a9b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-6941b-869c44b684 namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: c1f19fef-d8e1-497e-86db-74002bd5c9be kind: Event lastTimestamp: "2026-04-17T17:04:54Z" message: 'Created pod: switch-graph-6941b-869c44b684-24nwx' metadata: creationTimestamp: "2026-04-17T17:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:54Z" name: switch-graph-6941b-869c44b684.18a733c3e0f63290 namespace: kserve-ci-e2e-test resourceVersion: "26495" uid: ef48785b-dfe2-4811-8566-f13ac10a8836 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 3893ab55-dfaf-4599-9d48-e88419ec7cb6 kind: Event lastTimestamp: "2026-04-17T17:04:54Z" message: Scaled up replica set switch-graph-6941b-869c44b684 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:04:54Z" name: switch-graph-6941b.18a733c3dfe49da1 namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: 724757af-fab9-46ed-9ade-5a021b64bceb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:04:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "26504" uid: 1e74b45d-28b4-4148-9c3f-16542ab17ddd kind: Event lastTimestamp: "2026-04-17T17:04:54Z" message: InferenceGraph [switch-graph-6941b] is Ready metadata: creationTimestamp: "2026-04-17T17:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:04:54Z" name: switch-graph-6941b.18a733c3e4b2653f namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: 4baf0a4b-d861-47db-95c5-eb5861e24cfd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "26489" uid: 15fce5bb-4046-40e6-88c8-4b4691858285 kind: Event lastTimestamp: "2026-04-17T17:05:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:09Z" name: switch-graph-6941b.18a733c75e5c2d19 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: e8c90f55-1c76-4ac3-801d-e2c448ae436a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "26489" uid: 15fce5bb-4046-40e6-88c8-4b4691858285 kind: Event lastTimestamp: "2026-04-17T17:05:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:09Z" name: switch-graph-6941b.18a733c75ecf3e29 namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 4f4a0c39-a7bb-4cb8-a0e7-626e0417f1e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: 15fce5bb-4046-40e6-88c8-4b4691858285 kind: Event lastTimestamp: "2026-04-17T17:05:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:24Z" name: switch-graph-6941b.18a733caddd8917b namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: ed7ade33-c285-474e-b271-131cc711b872 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6941b namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: 15fce5bb-4046-40e6-88c8-4b4691858285 kind: Event lastTimestamp: "2026-04-17T17:05:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:05:24Z" name: switch-graph-6941b.18a733cade1c610a namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: a86e93c1-ce14-4f8f-b0a0-fedc7289eb57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:43:04.061542Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9e0ff-7bdf4d4867-lckwb namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 1e7495f8-7dea-426d-94a0-0d3ccc87fbee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9e0ff-7bdf4d4867-lckwb to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:43:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:43:04Z" name: switch-graph-9e0ff-7bdf4d4867-lckwb.18a73292ca4c1a48 namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: c22cd592-bd33-44d2-8be4-1d93555fbb89 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:04Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9e0ff-7bdf4d4867-lckwb namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 1e7495f8-7dea-426d-94a0-0d3ccc87fbee kind: Event lastTimestamp: "2026-04-17T16:43:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-9e0ff-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:04Z" name: switch-graph-9e0ff-7bdf4d4867-lckwb.18a73292d78c7275 namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: c8a91277-9cb6-4705-9e00-19a28f83a43e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:05Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9e0ff-7bdf4d4867-lckwb namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 1e7495f8-7dea-426d-94a0-0d3ccc87fbee kind: Event lastTimestamp: "2026-04-17T16:43:05Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:43:05Z" name: switch-graph-9e0ff-7bdf4d4867-lckwb.18a7329306e7971c namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: a0ab569e-add9-40be-8acb-4c9cdd16fc8c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9e0ff} kind: Pod name: switch-graph-9e0ff-7bdf4d4867-lckwb namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 1e7495f8-7dea-426d-94a0-0d3ccc87fbee kind: Event lastTimestamp: "2026-04-17T16:43:05Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" metadata: creationTimestamp: "2026-04-17T16:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:05Z" name: switch-graph-9e0ff-7bdf4d4867-lckwb.18a732930815c8e0 namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 00fc2b17-c532-4c5a-95f9-188a7375484a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9e0ff} kind: Pod name: switch-graph-9e0ff-7bdf4d4867-lckwb namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 1e7495f8-7dea-426d-94a0-0d3ccc87fbee kind: Event lastTimestamp: "2026-04-17T16:43:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" in 2.216s (2.216s including waiting). Image size: 216233549 bytes.' metadata: creationTimestamp: "2026-04-17T16:43:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:07Z" name: switch-graph-9e0ff-7bdf4d4867-lckwb.18a732938c2ef1e5 namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: 3566f3f1-e3a5-4ea3-99dd-d17a8dff2833 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9e0ff} kind: Pod name: switch-graph-9e0ff-7bdf4d4867-lckwb namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 1e7495f8-7dea-426d-94a0-0d3ccc87fbee kind: Event lastTimestamp: "2026-04-17T16:43:07Z" message: 'Created container: switch-graph-9e0ff' metadata: creationTimestamp: "2026-04-17T16:43:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:07Z" name: switch-graph-9e0ff-7bdf4d4867-lckwb.18a732938f54204b namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 82f504ab-5162-4c0b-82e1-1b346ddabe7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9e0ff} kind: Pod name: switch-graph-9e0ff-7bdf4d4867-lckwb namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 1e7495f8-7dea-426d-94a0-0d3ccc87fbee kind: Event lastTimestamp: "2026-04-17T16:43:07Z" message: Started container switch-graph-9e0ff metadata: creationTimestamp: "2026-04-17T16:43:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:07Z" name: switch-graph-9e0ff-7bdf4d4867-lckwb.18a732938fc57972 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: 3609d1f1-06fd-4c8a-bcf9-af2f1ef472c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9e0ff} kind: Pod name: switch-graph-9e0ff-7bdf4d4867-lckwb namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 1e7495f8-7dea-426d-94a0-0d3ccc87fbee kind: Event lastTimestamp: "2026-04-17T16:43:18Z" message: Stopping container switch-graph-9e0ff metadata: creationTimestamp: "2026-04-17T16:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:18Z" name: switch-graph-9e0ff-7bdf4d4867-lckwb.18a732961ba7b069 namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: b1044cde-3b4e-4786-99a2-495edec12f2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9e0ff} kind: Pod name: switch-graph-9e0ff-7bdf4d4867-lckwb namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 1e7495f8-7dea-426d-94a0-0d3ccc87fbee kind: Event lastTimestamp: "2026-04-17T16:43:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:44Z" name: switch-graph-9e0ff-7bdf4d4867-lckwb.18a7329650d7be41 namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: fa9368d3-1a06-4126-be79-3f372bf3543e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9e0ff-7bdf4d4867 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: d6bf321e-8518-44f0-822f-4f38e7aa2a21 kind: Event lastTimestamp: "2026-04-17T16:43:04Z" message: 'Created pod: switch-graph-9e0ff-7bdf4d4867-lckwb' metadata: creationTimestamp: "2026-04-17T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:04Z" name: switch-graph-9e0ff-7bdf4d4867.18a73292c9e8e3e1 namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 4c84d826-5fcc-4437-8a3a-73322022f0f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9e0ff namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: fac87ca4-a96b-45c6-8ca6-1e28c0df7bfc kind: Event lastTimestamp: "2026-04-17T16:43:04Z" message: Scaled up replica set switch-graph-9e0ff-7bdf4d4867 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:04Z" name: switch-graph-9e0ff.18a73292c8d8faf8 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 752d0494-3809-41ad-96fa-08df5e76d1a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9e0ff namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: bb42295a-8bb5-47d3-973c-6a4a1962fb3e kind: Event lastTimestamp: "2026-04-17T16:43:04Z" message: InferenceGraph [switch-graph-9e0ff] is Ready metadata: creationTimestamp: "2026-04-17T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:04Z" name: switch-graph-9e0ff.18a73292cdd650ff namespace: kserve-ci-e2e-test resourceVersion: "14933" uid: b027a4f1-e5d6-4117-965e-54230aa3cda2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:44:18.598570Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ce4d4-66ccfb86c4-sd5b6 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: cbc24bb6-ef54-446f-b031-2149f3fe5158 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ce4d4-66ccfb86c4-sd5b6 to ip-10-0-138-137.ec2.internal metadata: creationTimestamp: "2026-04-17T16:44:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:44:18Z" name: switch-graph-ce4d4-66ccfb86c4-sd5b6.18a732a4250cf12b namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 0b2b1783-254e-4e61-8fb6-694c4b17d68a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5bdbdf865c-7hctg source: {} type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:44:18Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ce4d4-66ccfb86c4-sd5b6 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: cbc24bb6-ef54-446f-b031-2149f3fe5158 kind: Event lastTimestamp: "2026-04-17T16:52:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-ce4d4-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:48Z" name: switch-graph-ce4d4-66ccfb86c4-sd5b6.18a732a42fd4d220 namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: bdb3667a-efbf-47c0-b03f-6f7f4adf51b0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:19Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ce4d4-66ccfb86c4-sd5b6 namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: cbc24bb6-ef54-446f-b031-2149f3fe5158 kind: Event lastTimestamp: "2026-04-17T16:44:19Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:44:19Z" name: switch-graph-ce4d4-66ccfb86c4-sd5b6.18a732a461eb788b namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: b9cfad90-86e6-4773-b555-906a2ec00c4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce4d4} kind: Pod name: switch-graph-ce4d4-66ccfb86c4-sd5b6 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: cbc24bb6-ef54-446f-b031-2149f3fe5158 kind: Event lastTimestamp: "2026-04-17T16:44:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T16:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:44:19Z" name: switch-graph-ce4d4-66ccfb86c4-sd5b6.18a732a4633e7fea namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 97b6181a-b317-452a-bc6f-76bc42059490 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce4d4} kind: Pod name: switch-graph-ce4d4-66ccfb86c4-sd5b6 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: cbc24bb6-ef54-446f-b031-2149f3fe5158 kind: Event lastTimestamp: "2026-04-17T16:44:19Z" message: 'Created container: switch-graph-ce4d4' metadata: creationTimestamp: "2026-04-17T16:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:44:19Z" name: switch-graph-ce4d4-66ccfb86c4-sd5b6.18a732a46690692d namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 6c35cb93-66a8-4746-90cc-e48ae82cddf8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce4d4} kind: Pod name: switch-graph-ce4d4-66ccfb86c4-sd5b6 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: cbc24bb6-ef54-446f-b031-2149f3fe5158 kind: Event lastTimestamp: "2026-04-17T16:44:19Z" message: Started container switch-graph-ce4d4 metadata: creationTimestamp: "2026-04-17T16:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:44:19Z" name: switch-graph-ce4d4-66ccfb86c4-sd5b6.18a732a466fd0816 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: eed191b7-6d8c-43bf-835f-15fc9833c1c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce4d4} kind: Pod name: switch-graph-ce4d4-66ccfb86c4-sd5b6 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: cbc24bb6-ef54-446f-b031-2149f3fe5158 kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Stopping container switch-graph-ce4d4 metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: switch-graph-ce4d4-66ccfb86c4-sd5b6.18a73317736777f5 namespace: kserve-ci-e2e-test resourceVersion: "19596" uid: 7c060151-13f5-4671-864a-d85c1567389a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:52:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce4d4} kind: Pod name: switch-graph-ce4d4-66ccfb86c4-sd5b6 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: cbc24bb6-ef54-446f-b031-2149f3fe5158 kind: Event lastTimestamp: "2026-04-17T16:53:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:52:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:01Z" name: switch-graph-ce4d4-66ccfb86c4-sd5b6.18a733180f02298b namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: ca335fcf-3073-4b48-9af9-4458623f9b93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-137.ec2.internal source: component: kubelet host: ip-10-0-138-137.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ce4d4-66ccfb86c4 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 77a4a29c-1b2e-453b-9208-fd5750f35145 kind: Event lastTimestamp: "2026-04-17T16:44:18Z" message: 'Created pod: switch-graph-ce4d4-66ccfb86c4-sd5b6' metadata: creationTimestamp: "2026-04-17T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:18Z" name: switch-graph-ce4d4-66ccfb86c4.18a732a424a92ca6 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: b88d2834-04a4-44a8-a81c-2b216026b83b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: cf417af9-5f99-4af4-9239-b0840a1d4ce7 kind: Event lastTimestamp: "2026-04-17T16:44:18Z" message: Scaled up replica set switch-graph-ce4d4-66ccfb86c4 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:18Z" name: switch-graph-ce4d4.18a732a423a9c484 namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: b3285a87-27ff-4c22-b781-aed6cef8669b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: 9c8a5ee7-cc48-4e35-81e8-26cc265b07b1 kind: Event lastTimestamp: "2026-04-17T16:44:18Z" message: InferenceGraph [switch-graph-ce4d4] is Ready metadata: creationTimestamp: "2026-04-17T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:18Z" name: switch-graph-ce4d4.18a732a4291901bc namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 769714cf-44e4-4065-8114-b36e22f44377 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:44:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: ba0a8ab1-1631-4cbd-8115-2e72c145fbae kind: Event lastTimestamp: "2026-04-17T16:45:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:03Z" name: switch-graph-ce4d4.18a732a7a209fc09 namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: fe3e8595-fc91-4c6e-8ac6-78ac55afda39 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:44:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ce4d4 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: ba0a8ab1-1631-4cbd-8115-2e72c145fbae kind: Event lastTimestamp: "2026-04-17T16:45:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:03Z" name: switch-graph-ce4d4.18a732a7a26f7aa7 namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 2259828b-4c5f-4829-80ac-87913c51d41d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33700"