--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:53:49.576014Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: 9a400003-12aa-4880-a99e-89a278351fee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-2d7b2-5475c4b8c9-gxkpr to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:53:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:53:49Z" name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr.18a900d1eba570be namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: 4615ecae-a875-4a51-a2b0-e2ae59f024d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:49Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: 9a400003-12aa-4880-a99e-89a278351fee kind: Event lastTimestamp: "2026-04-23T13:53:49Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:53:49Z" name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr.18a900d2045388c2 namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: fa42d368-2f73-4f2b-a380-5fd119e9bc20 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2d7b2} kind: Pod name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 9a400003-12aa-4880-a99e-89a278351fee kind: Event lastTimestamp: "2026-04-23T13:53:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T13:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:50Z" name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr.18a900d20610b52a namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 26686793-9765-4c73-82ef-70f658c51b04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2d7b2} kind: Pod name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 9a400003-12aa-4880-a99e-89a278351fee kind: Event lastTimestamp: "2026-04-23T13:53:50Z" message: 'Created container: ensemble-graph-2d7b2' metadata: creationTimestamp: "2026-04-23T13:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:50Z" name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr.18a900d208f9f645 namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: 3f90b988-f01b-41e1-a466-1ffd8f705c4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2d7b2} kind: Pod name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 9a400003-12aa-4880-a99e-89a278351fee kind: Event lastTimestamp: "2026-04-23T13:53:50Z" message: Started container ensemble-graph-2d7b2 metadata: creationTimestamp: "2026-04-23T13:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:50Z" name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr.18a900d2097bdcda namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 878570c5-2d0c-4962-abb3-1959c2cac804 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2d7b2} kind: Pod name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 9a400003-12aa-4880-a99e-89a278351fee kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: Stopping container ensemble-graph-2d7b2 metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:59Z" name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr.18a900d44230b3be namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: ee202569-d7e9-493d-802e-7cd2e323de12 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:54:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2d7b2} kind: Pod name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 9a400003-12aa-4880-a99e-89a278351fee kind: Event lastTimestamp: "2026-04-23T13:54:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:26Z" name: ensemble-graph-2d7b2-5475c4b8c9-gxkpr.18a900d4a0ef3a1e namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: e0ec158f-60f2-486e-a7c3-975dab6f1cbe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-2d7b2-5475c4b8c9 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 75913234-7017-4379-82dd-f8f03e0bb87c kind: Event lastTimestamp: "2026-04-23T13:53:49Z" message: 'Created pod: ensemble-graph-2d7b2-5475c4b8c9-gxkpr' metadata: creationTimestamp: "2026-04-23T13:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:49Z" name: ensemble-graph-2d7b2-5475c4b8c9.18a900d1eb3e4257 namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 5ecaadec-80cc-4516-8377-4d31916bf4b8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: 625b295c-52e6-47dc-b5a2-5ca49520581b kind: Event lastTimestamp: "2026-04-23T13:53:49Z" message: Scaled up replica set ensemble-graph-2d7b2-5475c4b8c9 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:49Z" name: ensemble-graph-2d7b2.18a900d1e9e7fd9b namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: eb31e3ed-e161-4c36-aca0-446b7702f78f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: 705e7706-631c-417d-a3f4-00f6afd38d9c kind: Event lastTimestamp: "2026-04-23T13:53:49Z" message: InferenceGraph [ensemble-graph-2d7b2] is Ready metadata: creationTimestamp: "2026-04-23T13:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:49Z" name: ensemble-graph-2d7b2.18a900d1f0a7c4dc namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 2032a7b7-8d88-499d-b12a-643ab303ab35 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:54:59.859734Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-37fbd-576848b778-mxpf4 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 9a070a55-54b0-48a6-b96a-58a34dd2fa30 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-37fbd-576848b778-mxpf4 to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:54:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:54:59Z" name: ensemble-graph-37fbd-576848b778-mxpf4.18a900e248e1d0ff namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 08253c4f-2ffa-40b7-a3bf-ea08c711c9a8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:00Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-37fbd-576848b778-mxpf4 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 9a070a55-54b0-48a6-b96a-58a34dd2fa30 kind: Event lastTimestamp: "2026-04-23T13:55:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-37fbd-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:55:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:55:00Z" name: ensemble-graph-37fbd-576848b778-mxpf4.18a900e2520daec5 namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: a3b10fe0-75ab-43f2-b579-33691e1f5706 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:00Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-37fbd-576848b778-mxpf4 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 9a070a55-54b0-48a6-b96a-58a34dd2fa30 kind: Event lastTimestamp: "2026-04-23T13:55:00Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:55:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:55:00Z" name: ensemble-graph-37fbd-576848b778-mxpf4.18a900e285655ebd namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: 36587311-e7d7-40e9-aecd-ded786fd8cb3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-37fbd} kind: Pod name: ensemble-graph-37fbd-576848b778-mxpf4 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 9a070a55-54b0-48a6-b96a-58a34dd2fa30 kind: Event lastTimestamp: "2026-04-23T13:55:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T13:55:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:55:00Z" name: ensemble-graph-37fbd-576848b778-mxpf4.18a900e286a94c07 namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 926fadb0-d3ac-4566-89d7-3e4255e3c405 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-37fbd} kind: Pod name: ensemble-graph-37fbd-576848b778-mxpf4 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 9a070a55-54b0-48a6-b96a-58a34dd2fa30 kind: Event lastTimestamp: "2026-04-23T13:55:00Z" message: 'Created container: ensemble-graph-37fbd' metadata: creationTimestamp: "2026-04-23T13:55:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:55:00Z" name: ensemble-graph-37fbd-576848b778-mxpf4.18a900e289861722 namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 319c4e66-ce1b-48e6-a09d-796ce3888310 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-37fbd} kind: Pod name: ensemble-graph-37fbd-576848b778-mxpf4 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 9a070a55-54b0-48a6-b96a-58a34dd2fa30 kind: Event lastTimestamp: "2026-04-23T13:55:00Z" message: Started container ensemble-graph-37fbd metadata: creationTimestamp: "2026-04-23T13:55:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:55:00Z" name: ensemble-graph-37fbd-576848b778-mxpf4.18a900e289f7bbcf namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 5c9c1c3c-c1cd-4461-b3ea-c2d0d3e594c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-37fbd} kind: Pod name: ensemble-graph-37fbd-576848b778-mxpf4 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 9a070a55-54b0-48a6-b96a-58a34dd2fa30 kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: Stopping container ensemble-graph-37fbd metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:14Z" name: ensemble-graph-37fbd-576848b778-mxpf4.18a901557529498f namespace: kserve-ci-e2e-test resourceVersion: "24561" uid: b3bba435-363e-4b10-9ba6-3b11e99cee77 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:03:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-37fbd} kind: Pod name: ensemble-graph-37fbd-576848b778-mxpf4 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 9a070a55-54b0-48a6-b96a-58a34dd2fa30 kind: Event lastTimestamp: "2026-04-23T14:03:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T14:03:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:42Z" name: ensemble-graph-37fbd-576848b778-mxpf4.18a9015622e52d36 namespace: kserve-ci-e2e-test resourceVersion: "24895" uid: c937e7b6-f174-4adf-bb6a-aaf96d5fd0c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-37fbd-576848b778 namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 7c3e6ddd-f6dc-4a14-8404-d1711669374b kind: Event lastTimestamp: "2026-04-23T13:54:59Z" message: 'Created pod: ensemble-graph-37fbd-576848b778-mxpf4' metadata: creationTimestamp: "2026-04-23T13:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:59Z" name: ensemble-graph-37fbd-576848b778.18a900e24855971f namespace: kserve-ci-e2e-test resourceVersion: "21016" uid: 5ff77ebc-c457-4b5d-85f1-34beb7d9e5b4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-37fbd namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: dddc671c-12f8-4835-ae6b-d6ff6eef2a95 kind: Event lastTimestamp: "2026-04-23T13:54:59Z" message: Scaled up replica set ensemble-graph-37fbd-576848b778 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:59Z" name: ensemble-graph-37fbd.18a900e247a7dbcd namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 4153f3a2-688b-41f2-b8e2-a54f587a4abb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-37fbd namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: a0265609-05b8-4914-87ff-56e15affd805 kind: Event lastTimestamp: "2026-04-23T13:54:59Z" message: InferenceGraph [ensemble-graph-37fbd] is Ready metadata: creationTimestamp: "2026-04-23T13:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:59Z" name: ensemble-graph-37fbd.18a900e24b922f25 namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: b515aaae-d9a9-4d50-91dd-1efa54d46adf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-37fbd namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: 70634a83-c738-4fa9-8d77-ba109a6ee000 kind: Event lastTimestamp: "2026-04-23T13:55:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:29Z" name: ensemble-graph-37fbd.18a900e5c62c6004 namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 29fce2fb-8c1c-4e68-ac71-d94dadfac42f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-37fbd namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: 70634a83-c738-4fa9-8d77-ba109a6ee000 kind: Event lastTimestamp: "2026-04-23T13:55:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:29Z" name: ensemble-graph-37fbd.18a900e5c6831256 namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: ce660dc1-9290-4510-a5aa-e937b81f8eb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:53:25.368735Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900cc48c79a55 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 5c8f2197-c524-4834-81be-2c9a1bffba17 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900cc61c2b1a3 namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: 670b5950-5c8b-49ea-9718-b135360fbdc6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900cc6301235d namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: a79e95bc-eda3-4c65-b876-dc847f98900e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900cc6602838b namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: fcaa160b-f3ac-4453-be6a-a1120a9535d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900cc666e2bc3 namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: 9bf31048-a289-46d2-bb38-e89f87a1455a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900cc667ba9e2 namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: dffa2fe0-faf6-4f4e-80e8-657e8d2ea4a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900cc695d5687 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 15fc80d9-c510-4fcc-9b41-20026db73e85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900cc69c7d90d namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: 919d3839-da3c-4588-848f-8926d12ad26c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:53:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:54:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:53:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:03Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900ccb18204ae namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 1cd248ed-1c33-457e-b8af-41447d7a56d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:35Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900dc99009759 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: cdccb3b6-81f9-49a9-b53d-3bc8faa90f16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:35Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900dc9900f6ba namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 21fdc8c6-4649-432a-9432-2eb2dcc3d6d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 38b4a054-ea79-474e-ab9a-7cdac8139d19 kind: Event lastTimestamp: "2026-04-23T13:54:38Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:38Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg.18a900dd39b53275 namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: d0683ee5-9f10-4896-b7a3-88e11f1346bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1cfcc-predictor-655bf67bfd namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 4a5df487-24ee-438e-90e4-7ec01d96b87e kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: 'Created pod: error-404-isvc-1cfcc-predictor-655bf67bfd-prkpg' metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor-655bf67bfd.18a900cc4854f606 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 751af5eb-5270-4ce1-9116-1506a4cfbf54 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: 16b5172b-902f-4a72-9cf9-5bbe52ea0102 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Scaled up replica set error-404-isvc-1cfcc-predictor-655bf67bfd from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc-predictor.18a900cc46ab483b namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: 47242107-19b8-4823-924a-07c5e1e8c7b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: cf77c74c-7599-4841-97f4-5d54b9791a0e kind: Event lastTimestamp: "2026-04-23T13:53:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:40Z" name: error-404-isvc-1cfcc-predictor.18a900cfc5474e27 namespace: kserve-ci-e2e-test resourceVersion: "19963" uid: a4809453-252f-4a12-bbb8-d22fa2f92ab4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: cf77c74c-7599-4841-97f4-5d54b9791a0e kind: Event lastTimestamp: "2026-04-23T13:53:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:40Z" name: error-404-isvc-1cfcc-predictor.18a900cfc5db52b3 namespace: kserve-ci-e2e-test resourceVersion: "19965" uid: 52cb39bc-a67b-4ec9-8c89-b58b803109e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:53:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: cf77c74c-7599-4841-97f4-5d54b9791a0e kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:25Z" name: error-404-isvc-1cfcc-predictor.18a900d344b09bd8 namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: 63143bfb-452a-4c92-bbdc-7a28aaff885c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:53:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: cf77c74c-7599-4841-97f4-5d54b9791a0e kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:25Z" name: error-404-isvc-1cfcc-predictor.18a900d345f96c65 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: d0c9d716-0678-47d0-9ac7-ddb7aa61dfde reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: e43aa0e3-8d78-42ff-9174-00cc63bf61c5 kind: Event lastTimestamp: "2026-04-23T13:54:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:13Z" name: error-404-isvc-1cfcc.18a900cc4687df05 namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: 4427729c-5f11-4f63-98b7-1d91cad63d4c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: e43aa0e3-8d78-42ff-9174-00cc63bf61c5 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-1cfcc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1cfcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc.18a900cc46dc37cb namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: d1f58903-9f54-458c-9000-dcb609ebbfe3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: e43aa0e3-8d78-42ff-9174-00cc63bf61c5 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1cfcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-1cfcc.18a900cc46dc9237 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 7f2467bf-5d2c-4173-871e-a4c10359324c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: e43aa0e3-8d78-42ff-9174-00cc63bf61c5 kind: Event lastTimestamp: "2026-04-23T13:54:13Z" message: InferenceService [error-404-isvc-1cfcc] is Ready metadata: creationTimestamp: "2026-04-23T13:54:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:13Z" name: error-404-isvc-1cfcc.18a900d76d2364b2 namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: 7122b12b-35b7-40c6-b368-7777c513d810 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:52:49.586640Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19358" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2d7b2-predictor-595799759b-8gwcl to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:52:49Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900c3f4003e03 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b110b948-1074-422d-a2c7-2b0d5df71ba8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:52:50Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900c40ceced7e namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: 4d2fffc9-abe5-4748-b795-258ce10909ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900c40e591ae6 namespace: kserve-ci-e2e-test resourceVersion: "19384" uid: 15d95f57-be1a-494e-b758-d3f64e9c36b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900c411076080 namespace: kserve-ci-e2e-test resourceVersion: "19385" uid: d9e73518-1a21-45d0-820d-edbd85c930b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900c41174a7eb namespace: kserve-ci-e2e-test resourceVersion: "19386" uid: 3ec9f0c9-3902-4209-a1fb-9304c2944795 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900c4117fe2b3 namespace: kserve-ci-e2e-test resourceVersion: "19387" uid: f333e8f8-49a1-4ff5-99e7-1a43ce1f7e70 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900c414953a14 namespace: kserve-ci-e2e-test resourceVersion: "19388" uid: 700096bd-2fc0-4a84-af93-442d50b8cb86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900c41500e6ff namespace: kserve-ci-e2e-test resourceVersion: "19389" uid: 9b0c208a-da05-453a-ba81-89f3cfd1d43d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:52:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:53:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:27Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900c44939c345 namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: a9c7569c-ab43-4abf-8c59-dfd0ce23f186 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:59Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900d453de8fee namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: 81c83473-2082-4b48-bba4-622236ab1c44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:59Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900d453df0523 namespace: kserve-ci-e2e-test resourceVersion: "20257" uid: 515baa74-c33b-4a60-a972-eb339db4d29a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 31bac30d-bf67-403f-b7e9-a5577ec06b89 kind: Event lastTimestamp: "2026-04-23T13:54:02Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:02Z" name: error-404-isvc-2d7b2-predictor-595799759b-8gwcl.18a900d4d1514162 namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 526601d4-db93-47eb-aa99-130218be438e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2d7b2-predictor-595799759b namespace: kserve-ci-e2e-test resourceVersion: "19351" uid: f67624ac-8f5c-4128-92b9-b695a986b4b0 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: 'Created pod: error-404-isvc-2d7b2-predictor-595799759b-8gwcl' metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:52:49Z" name: error-404-isvc-2d7b2-predictor-595799759b.18a900c3f394c47c namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 98b0d8b1-1f19-4340-ab93-770aa1e011c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: 8d3ef4b0-9db6-4b74-a3de-3428f0f32fd4 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: Scaled up replica set error-404-isvc-2d7b2-predictor-595799759b from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:52:49Z" name: error-404-isvc-2d7b2-predictor.18a900c3f061cb23 namespace: kserve-ci-e2e-test resourceVersion: "19352" uid: 12e6a409-9a97-4b96-8773-f8c88b11e8ee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:53:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19350" uid: e0cfd87b-5b21-4e8e-826a-4e5bc38a1eba kind: Event lastTimestamp: "2026-04-23T13:53:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:19Z" name: error-404-isvc-2d7b2-predictor.18a900c76f02479a namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 604a743f-e81a-472f-b33d-294ad830d507 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:53:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19350" uid: e0cfd87b-5b21-4e8e-826a-4e5bc38a1eba kind: Event lastTimestamp: "2026-04-23T13:53:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:19Z" name: error-404-isvc-2d7b2-predictor.18a900c76f6c5fd6 namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: d9545d76-def8-49b7-bd5a-ede958936d46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:53:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19528" uid: e0cfd87b-5b21-4e8e-826a-4e5bc38a1eba kind: Event lastTimestamp: "2026-04-23T13:53:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:49Z" name: error-404-isvc-2d7b2-predictor.18a900ce6ce1c5f5 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: a9af2f25-abbe-4ea8-b484-c8fe4fa22fcc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:53:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19528" uid: e0cfd87b-5b21-4e8e-826a-4e5bc38a1eba kind: Event lastTimestamp: "2026-04-23T13:53:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:49Z" name: error-404-isvc-2d7b2-predictor.18a900ce6d494c73 namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: 9295f546-36c2-49d8-a779-968745a5586a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: 20dec5d1-3e4d-41bf-8cdb-bcc11952d9e9 kind: Event lastTimestamp: "2026-04-23T13:53:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:37Z" name: error-404-isvc-2d7b2.18a900c3f06c2b07 namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: 8b712883-567c-4525-a280-51aae68807b0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: 20dec5d1-3e4d-41bf-8cdb-bcc11952d9e9 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: 'Failed to update status for InferenceService "error-404-isvc-2d7b2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2d7b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:52:49Z" name: error-404-isvc-2d7b2.18a900c3f0f034bc namespace: kserve-ci-e2e-test resourceVersion: "19355" uid: 2436b485-88fb-4d0c-bba8-64c5a4833921 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: 20dec5d1-3e4d-41bf-8cdb-bcc11952d9e9 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2d7b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:52:49Z" name: error-404-isvc-2d7b2.18a900c3f0f07a54 namespace: kserve-ci-e2e-test resourceVersion: "19356" uid: b6656c65-08ac-47d0-9dc5-484355f87e97 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: 20dec5d1-3e4d-41bf-8cdb-bcc11952d9e9 kind: Event lastTimestamp: "2026-04-23T13:53:37Z" message: InferenceService [error-404-isvc-2d7b2] is Ready metadata: creationTimestamp: "2026-04-23T13:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:37Z" name: error-404-isvc-2d7b2.18a900cf04f5a502 namespace: kserve-ci-e2e-test resourceVersion: "19924" uid: 8a58740a-ea84-4dc4-b5d9-8d485941dcc6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:53:59.957613Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20260" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-37fbd-predictor-7544db6574-c755q to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:53:59Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a900d4566ffeb1 namespace: kserve-ci-e2e-test resourceVersion: "20264" uid: a5a3b0a8-79f3-41df-a8b3-6de9f444d624 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:54:00Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a900d46f417bce namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 3e75d158-5feb-4d12-8a98-89d7e5c00899 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a900d4715d9619 namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 9a047780-5b9c-40b8-9cef-5e1c4a90b3c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a900d474121824 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: 8b730442-c148-45f6-acec-1919a2b7bede reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a900d4747f56eb namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: c4b9eceb-759d-4785-b920-7cc4b68b81e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a900d4748ad58e namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: df70b176-647a-4130-9d90-29ae4c7e589e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a900d476de2629 namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: fe3c2707-7565-40ff-898a-0cbe0162599c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a900d4774b5244 namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: 53ae39f7-9842-4894-883d-d52bb6cc3a03 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T13:54:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:38Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a900d4df0fe640 namespace: kserve-ci-e2e-test resourceVersion: "20810" uid: 848a8564-279f-4c95-b42b-3d9e0e8f8c03 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:14Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a90155809fb7f1 namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: bd4d8aef-640e-489a-843d-de1f926917cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-37fbd-predictor-7544db6574-c755q namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 014980aa-a818-4b54-8ae1-7d91c7a3e843 kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:14Z" name: error-404-isvc-37fbd-predictor-7544db6574-c755q.18a9015580a033c6 namespace: kserve-ci-e2e-test resourceVersion: "24620" uid: 5d97a8fa-fef7-4c77-b173-99cc9c7fad49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-37fbd-predictor-7544db6574 namespace: kserve-ci-e2e-test resourceVersion: "20251" uid: cbf7010b-4294-4b77-a21f-152dc841680e kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: 'Created pod: error-404-isvc-37fbd-predictor-7544db6574-c755q' metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:59Z" name: error-404-isvc-37fbd-predictor-7544db6574.18a900d455fdcae3 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: dea95ca3-4d58-41af-9307-94409a1bdaca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: b0c73456-f6ce-41ee-93f7-efd32aebaa2e kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: Scaled up replica set error-404-isvc-37fbd-predictor-7544db6574 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:59Z" name: error-404-isvc-37fbd-predictor.18a900d453e49da6 namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: ab5240e3-312d-4389-9b0a-a3f9608df923 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 56114863-e22f-4b0d-b609-83b63bdefb0b kind: Event lastTimestamp: "2026-04-23T13:54:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:14Z" name: error-404-isvc-37fbd-predictor.18a900d7d2742727 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 4e7a3e2c-349a-4b43-8123-b948a538cef7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 56114863-e22f-4b0d-b609-83b63bdefb0b kind: Event lastTimestamp: "2026-04-23T13:54:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:14Z" name: error-404-isvc-37fbd-predictor.18a900d7d2ddbb97 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: d2e5ec1a-b13b-4bbf-9e1a-eccc56d88c88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:54:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 56114863-e22f-4b0d-b609-83b63bdefb0b kind: Event lastTimestamp: "2026-04-23T13:54:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:59Z" name: error-404-isvc-37fbd-predictor.18a900db517f0039 namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 543e4482-7ab6-4dea-b4ad-3f5fc7b36c1b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:54:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 56114863-e22f-4b0d-b609-83b63bdefb0b kind: Event lastTimestamp: "2026-04-23T13:54:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:59Z" name: error-404-isvc-37fbd-predictor.18a900db51e04876 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: dea4599f-c0fa-4f06-857a-b5ae4a49e774 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-37fbd namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: cadcdee1-1660-4124-bb68-3fd8a8030957 kind: Event lastTimestamp: "2026-04-23T13:54:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:48Z" name: error-404-isvc-37fbd.18a900d453afc274 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 6d96a565-271e-4041-8175-7e77333ba3ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-37fbd namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: cadcdee1-1660-4124-bb68-3fd8a8030957 kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: 'Failed to update status for InferenceService "error-404-isvc-37fbd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-37fbd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:59Z" name: error-404-isvc-37fbd.18a900d4544d1955 namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: 1e21fa26-c998-4c5d-8cad-5cf25ff0a865 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-37fbd namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: cadcdee1-1660-4124-bb68-3fd8a8030957 kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-37fbd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:59Z" name: error-404-isvc-37fbd.18a900d4544d6627 namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 118f2f64-e598-4234-a227-dbe90d189a8c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-37fbd namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: cadcdee1-1660-4124-bb68-3fd8a8030957 kind: Event lastTimestamp: "2026-04-23T13:54:48Z" message: InferenceService [error-404-isvc-37fbd] is Ready metadata: creationTimestamp: "2026-04-23T13:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:48Z" name: error-404-isvc-37fbd.18a900df9a6fcfec namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: f7bb9fd3-f178-4e7f-b470-a12b68c89753 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T14:03:14.811511Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24641" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T14:03:14Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90155864eaac5 namespace: kserve-ci-e2e-test resourceVersion: "24645" uid: c8a741d7-e94a-40a2-af4c-74c33df46783 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24651" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T14:03:15Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a901559f47f05d namespace: kserve-ci-e2e-test resourceVersion: "24663" uid: cc95b00b-03d9-438a-a2b9-87d7697cad58 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90155a0a1b34f namespace: kserve-ci-e2e-test resourceVersion: "24667" uid: e192039f-4ba4-45a6-b29a-3ac008d1b4fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90155a36b48b1 namespace: kserve-ci-e2e-test resourceVersion: "24670" uid: 4a413310-c8a2-4828-9e36-d1a343275173 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90155a3d826e3 namespace: kserve-ci-e2e-test resourceVersion: "24672" uid: e9f8354c-6da9-4ab5-8db8-a01c194723f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90155a3e2c7f8 namespace: kserve-ci-e2e-test resourceVersion: "24673" uid: ab0c92df-e969-49b4-a339-d1ec2b22181d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90155a6804184 namespace: kserve-ci-e2e-test resourceVersion: "24674" uid: a111475b-7d31-45f1-b07d-a7fc5846f5c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90155a6ebcf9c namespace: kserve-ci-e2e-test resourceVersion: "24675" uid: e68da71a-56ca-4285-9ce9-b5eb6832b367 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:03:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:03:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T14:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:52Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90155fdbf7da9 namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: bcfb5eb0-ac6f-41e7-b64a-f678b84ba04d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:04:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:04:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:24Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90165d91ecbed namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: ee9d38c7-b620-4dec-a728-e251ed774314 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:04:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:04:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:24Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a90165d9200aca namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: 7a505f07-d963-4542-a4f8-79ec2b55ecf3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: fca22878-ab4a-4c89-8f94-e4a1e5a08647 kind: Event lastTimestamp: "2026-04-23T14:04:27Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T14:04:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:27Z" name: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb.18a9016685e698cd namespace: kserve-ci-e2e-test resourceVersion: "25596" uid: 894f60f0-5977-4c9a-8f9c-7188deb2e008 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5035e-predictor-6d8685dc6b namespace: kserve-ci-e2e-test resourceVersion: "24632" uid: 7340e68d-200e-412f-b4d6-8e445cc16f9b kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: 'Created pod: error-404-isvc-5035e-predictor-6d8685dc6b-q4hpb' metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:14Z" name: error-404-isvc-5035e-predictor-6d8685dc6b.18a9015585ef2233 namespace: kserve-ci-e2e-test resourceVersion: "24643" uid: 54c45a7a-7b3f-4f80-9419-f0e6e145022a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: 3f2e5ffd-eaeb-48c4-93a7-49e609e9dc99 kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: Scaled up replica set error-404-isvc-5035e-predictor-6d8685dc6b from 0 to 1 metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:14Z" name: error-404-isvc-5035e-predictor.18a901558393fd8b namespace: kserve-ci-e2e-test resourceVersion: "24635" uid: 9450d244-f3cb-4a76-8c62-c683c3b143c8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24631" uid: 99ed3d8f-8050-4c61-a497-9fcfb26387c2 kind: Event lastTimestamp: "2026-04-23T14:03:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:44Z" name: error-404-isvc-5035e-predictor.18a90159021605cd namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: 8a550ffc-c547-4a41-aee3-cd584ac56c22 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24631" uid: 99ed3d8f-8050-4c61-a497-9fcfb26387c2 kind: Event lastTimestamp: "2026-04-23T14:03:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:44Z" name: error-404-isvc-5035e-predictor.18a90159027dbd78 namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 9e57cedf-944e-4a8e-beef-acbff4d5d980 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24804" uid: 99ed3d8f-8050-4c61-a497-9fcfb26387c2 kind: Event lastTimestamp: "2026-04-23T14:04:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:14Z" name: error-404-isvc-5035e-predictor.18a9015ffffd7e6a namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: f418f0c1-4a4d-4907-b227-73bbd4b6cccf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24804" uid: 99ed3d8f-8050-4c61-a497-9fcfb26387c2 kind: Event lastTimestamp: "2026-04-23T14:04:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:14Z" name: error-404-isvc-5035e-predictor.18a9016000593c27 namespace: kserve-ci-e2e-test resourceVersion: "25339" uid: 8306c38a-e841-4b32-83bb-896404758931 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5035e namespace: kserve-ci-e2e-test resourceVersion: "24604" uid: 36b25c00-09dc-44b1-9f76-bbccaa91f0e0 kind: Event lastTimestamp: "2026-04-23T14:04:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:02Z" name: error-404-isvc-5035e.18a9015583607e3f namespace: kserve-ci-e2e-test resourceVersion: "25220" uid: 2a9f187e-4fec-4618-89a2-9fc5e1b4a4e4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5035e namespace: kserve-ci-e2e-test resourceVersion: "24604" uid: 36b25c00-09dc-44b1-9f76-bbccaa91f0e0 kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: 'Failed to update status for InferenceService "error-404-isvc-5035e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5035e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:03:14Z" name: error-404-isvc-5035e.18a90155840936b5 namespace: kserve-ci-e2e-test resourceVersion: "24637" uid: 38755e1c-29ad-4dcf-9eb4-5b894c9ede24 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5035e namespace: kserve-ci-e2e-test resourceVersion: "24604" uid: 36b25c00-09dc-44b1-9f76-bbccaa91f0e0 kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5035e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:03:14Z" name: error-404-isvc-5035e.18a9015584097217 namespace: kserve-ci-e2e-test resourceVersion: "24638" uid: c13167b7-179e-4d60-aa25-5c60c33634ab reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5035e namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: 36b25c00-09dc-44b1-9f76-bbccaa91f0e0 kind: Event lastTimestamp: "2026-04-23T14:04:02Z" message: InferenceService [error-404-isvc-5035e] is Ready metadata: creationTimestamp: "2026-04-23T14:04:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:02Z" name: error-404-isvc-5035e.18a90160b940fc68 namespace: kserve-ci-e2e-test resourceVersion: "25219" uid: c4f88dc6-62ee-4b75-bd92-9c7626b98e65 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:54:35.634149Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8d150-predictor-568b6759f8-b7rcq to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:54:35Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dca4ecbd72 namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 39be28f2-fb10-4c50-a326-9c3d0b096536 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8d150-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:35Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dcb14985b4 namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 125d247c-ed5a-4587-afeb-17c03274458b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20751" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:54:36Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dce1639bc1 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 0c8e240f-2974-47c4-83ae-7ecadb452bfa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dce2f706cc namespace: kserve-ci-e2e-test resourceVersion: "20778" uid: 986758a5-9b1d-4620-9d0c-2e6d5ae50c47 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dce6020ea3 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: e020960f-cedc-4bcf-a2be-955a8af1e277 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dce66c109e namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: c44f64e5-d5f4-460e-8390-59e1e054ecf2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dce6778cd9 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 78131573-b543-44d9-aacb-c4926f5138cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dce925ef28 namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: a8e57a8e-4cbd-4c09-85d5-057848d7b178 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dce993af96 namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: 9efaba3d-bcf4-45be-bfac-8d1c1824d752 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T13:55:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:55:14Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a900dd47769163 namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: dc64018d-672c-46ca-91f5-05dba1d4ff0e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:50Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a9015dd8c269de namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: 35295a5c-de0b-4be5-95e5-b92314b298ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 2cd36014-1cf4-473d-9b21-8a05cbcca2f7 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:50Z" name: error-404-isvc-8d150-predictor-568b6759f8-b7rcq.18a9015dd8c2f84f namespace: kserve-ci-e2e-test resourceVersion: "25052" uid: 5993b68c-bb6b-41ad-b9c5-c0550df4102d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8d150-predictor-568b6759f8 namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: 63a2bef5-2fa6-4dd4-a521-948139e45d5c kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: 'Created pod: error-404-isvc-8d150-predictor-568b6759f8-b7rcq' metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:35Z" name: error-404-isvc-8d150-predictor-568b6759f8.18a900dca45462ee namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: 63a4d031-fdfb-48a1-96f3-19535d43b7d1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: e53019fe-90e5-41d2-b55d-8555e3417c7b kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: Scaled up replica set error-404-isvc-8d150-predictor-568b6759f8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:35Z" name: error-404-isvc-8d150-predictor.18a900dca2b24eb5 namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: edfbfb37-39d6-4a44-9e6d-34efed9a6e51 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:54:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: dc1f7e31-cd61-47a4-a1a2-8a3d4831f343 kind: Event lastTimestamp: "2026-04-23T13:55:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:05Z" name: error-404-isvc-8d150-predictor.18a900e021487c64 namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: ca245493-8136-494e-84d2-13b2ee68ef64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:54:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: dc1f7e31-cd61-47a4-a1a2-8a3d4831f343 kind: Event lastTimestamp: "2026-04-23T13:55:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:05Z" name: error-404-isvc-8d150-predictor.18a900e021b350de namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: b8f48652-24ac-43f0-aec8-628669fb8631 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:55:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: dc1f7e31-cd61-47a4-a1a2-8a3d4831f343 kind: Event lastTimestamp: "2026-04-23T13:55:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:55:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:50Z" name: error-404-isvc-8d150-predictor.18a900e71fba2dba namespace: kserve-ci-e2e-test resourceVersion: "21495" uid: b796e71e-fc7d-4bcd-9674-552b76e4e11b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:55:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: dc1f7e31-cd61-47a4-a1a2-8a3d4831f343 kind: Event lastTimestamp: "2026-04-23T13:55:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:55:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:50Z" name: error-404-isvc-8d150-predictor.18a900e72026537b namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: 181e79e3-4466-40ff-8737-0109131a852d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d150 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 2ece2c76-10be-44fb-b7e8-1f1991419c12 kind: Event lastTimestamp: "2026-04-23T13:55:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:55:24Z" name: error-404-isvc-8d150.18a900dca2963b2e namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: 9b458ab3-9cfc-4c26-89d5-5a911ecfc8af reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d150 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 2ece2c76-10be-44fb-b7e8-1f1991419c12 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-8d150": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d150": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:35Z" name: error-404-isvc-8d150.18a900dca30f57fe namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: 033c6fa7-c34a-4941-ba7a-bd2e6aec6dae reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d150 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 2ece2c76-10be-44fb-b7e8-1f1991419c12 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d150": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:35Z" name: error-404-isvc-8d150.18a900dca30f95c4 namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: 8ff46380-08d5-40e5-af73-52f255208b75 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d150 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 2ece2c76-10be-44fb-b7e8-1f1991419c12 kind: Event lastTimestamp: "2026-04-23T13:55:24Z" message: InferenceService [error-404-isvc-8d150] is Ready metadata: creationTimestamp: "2026-04-23T13:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:55:24Z" name: error-404-isvc-8d150.18a900e8023baeb6 namespace: kserve-ci-e2e-test resourceVersion: "21256" uid: a2693958-1030-427c-afd3-f2d02c3a7f94 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:42:00.545155Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13857" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-94d25-predictor-f56cbdc6b-6r59g to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:42:00Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a9002cd61d1760 namespace: kserve-ci-e2e-test resourceVersion: "13863" uid: cb304e9e-a940-4b07-b870-5bbedcff302a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-94d25-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:00Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a9002cdfb62ede namespace: kserve-ci-e2e-test resourceVersion: "13872" uid: 1c2a99b2-fbf2-4844-87c1-43e372669143 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13869" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:01Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:42:01Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a9002d1285bb4a namespace: kserve-ci-e2e-test resourceVersion: "13913" uid: 8ed4f33c-8e61-46be-8a19-1234910a4372 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:01Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" metadata: creationTimestamp: "2026-04-23T13:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:01Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a9002d13d940f5 namespace: kserve-ci-e2e-test resourceVersion: "13915" uid: 42482716-dfc8-46ed-9c28-faa1c49254d7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:14Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" in 12.746s (12.746s including waiting). Image size: 1336675422 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:14Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a900300b9b69a9 namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: f9f7fc46-dc33-47d4-a813-7bbfceb1dd37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:14Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a900300ede0763 namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: ad9b0c63-cc5a-40a2-a2a8-e8d0d81b4aea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:14Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a900300f3b4489 namespace: kserve-ci-e2e-test resourceVersion: "14056" uid: 6e400d1b-0b5e-4f16-962f-560c7b6c7979 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:14Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T13:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:14Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a900300f491671 namespace: kserve-ci-e2e-test resourceVersion: "14057" uid: 54a425b3-8253-48e6-8e14-0f1b8de16692 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:16Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.265s (2.265s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:16Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a90030964d96fc namespace: kserve-ci-e2e-test resourceVersion: "14111" uid: 69d66a86-bccf-4948-9a37-7daa93623bdc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:16Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a900309935d76e namespace: kserve-ci-e2e-test resourceVersion: "14112" uid: 9d01b73b-b958-4036-aa76-8bc323c9f57f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:16Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a900309994464c namespace: kserve-ci-e2e-test resourceVersion: "14113" uid: 812ad6ae-e22d-4262-9325-930f2920bed8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:42:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:54Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a90030f4edb8db namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: da32be9d-3f4d-40c6-b4d3-deae023ba20f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:34Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a90042b9fd01f2 namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: a836ce6c-d64a-47bb-af45-896e9aa97267 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: 555b7e6f-53f2-4b5d-9b94-1286de763e29 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:34Z" name: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g.18a90042b9fd740d namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 9871b5e2-e6e1-4171-9376-32b8b0eade53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-94d25-predictor-f56cbdc6b namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: ef8e4bb1-a9ea-4e41-a828-ac3946129a15 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: 'Created pod: error-404-isvc-94d25-predictor-f56cbdc6b-6r59g' metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:00Z" name: error-404-isvc-94d25-predictor-f56cbdc6b.18a9002cd599b427 namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: d67dbd72-f9ec-4cfa-a007-3589e4add5ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "13846" uid: 577fb109-ba82-4336-804a-4d202e1b58e6 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: Scaled up replica set error-404-isvc-94d25-predictor-f56cbdc6b from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:00Z" name: error-404-isvc-94d25-predictor.18a9002ccf505e87 namespace: kserve-ci-e2e-test resourceVersion: "13849" uid: f9cc1e8d-72fe-4071-906d-834e3812536c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "13848" uid: 0582d058-78a6-44e9-aa6f-ab72cb2abdba kind: Event lastTimestamp: "2026-04-23T13:42:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:45Z" name: error-404-isvc-94d25-predictor.18a900304de59998 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 5daf441a-9f1c-44f0-9756-d3e4269c4ce9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "13848" uid: 0582d058-78a6-44e9-aa6f-ab72cb2abdba kind: Event lastTimestamp: "2026-04-23T13:42:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:45Z" name: error-404-isvc-94d25-predictor.18a900304e3dc2e8 namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 2443624c-68ef-4d33-8d8a-6db23fecf2be reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:43:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "14086" uid: 0582d058-78a6-44e9-aa6f-ab72cb2abdba kind: Event lastTimestamp: "2026-04-23T13:43:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:15Z" name: error-404-isvc-94d25-predictor.18a9003aca586513 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: e4fbb37d-34cd-4e81-b930-c13b0948c726 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:43:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "14086" uid: 0582d058-78a6-44e9-aa6f-ab72cb2abdba kind: Event lastTimestamp: "2026-04-23T13:43:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:15Z" name: error-404-isvc-94d25-predictor.18a9003acab8a492 namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: 81ae9a45-f782-46d6-a3c0-e233db3e37b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-94d25 namespace: kserve-ci-e2e-test resourceVersion: "13828" uid: 1220eae3-053b-4b11-b500-fa80452d239c kind: Event lastTimestamp: "2026-04-23T13:43:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:04Z" name: error-404-isvc-94d25.18a9002ccf57dc39 namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: 73beacb0-52c8-4780-b417-e37c9c48fd55 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-94d25 namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: 1220eae3-053b-4b11-b500-fa80452d239c kind: Event lastTimestamp: "2026-04-23T13:43:04Z" message: InferenceService [error-404-isvc-94d25] is Ready metadata: creationTimestamp: "2026-04-23T13:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:04Z" name: error-404-isvc-94d25.18a9003baff093f4 namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 09677b1b-a6ab-47c9-bcee-9f7de434c54b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T14:04:25.134682Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25533" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a90165e5e50d86 namespace: kserve-ci-e2e-test resourceVersion: "25538" uid: 2dfe2030-3f49-4d37-8bdd-8a046505a603 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25543" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a90165fe9f3fa7 namespace: kserve-ci-e2e-test resourceVersion: "25554" uid: 4c3f88d1-e854-44db-ab6c-24f7eff1eb0a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a9016600032430 namespace: kserve-ci-e2e-test resourceVersion: "25556" uid: 817e54b5-aeca-4f76-983c-c39dc043f839 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a90166042ca19a namespace: kserve-ci-e2e-test resourceVersion: "25559" uid: a95950b6-4cf6-4779-84b2-1214c2d115aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a90166049eb10f namespace: kserve-ci-e2e-test resourceVersion: "25560" uid: f9cfafba-85ca-4bd3-ba42-e703ccb72113 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a9016604abbee9 namespace: kserve-ci-e2e-test resourceVersion: "25561" uid: 1235db8e-2bc2-46b5-b845-5930ca1a42cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a9016607149bcb namespace: kserve-ci-e2e-test resourceVersion: "25562" uid: ad4c93b6-7bf4-4e54-bf44-3568824cece6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a9016607840436 namespace: kserve-ci-e2e-test resourceVersion: "25563" uid: 73dbe553-7834-4ac7-8c98-6d384cadb5d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:04:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:05:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T14:04:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:05:03Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a90166582d5c59 namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 14a9f1f8-4de9-4c99-87a8-5cc1c7b7f513 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:13:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:13:39Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a901e70ec14126 namespace: kserve-ci-e2e-test resourceVersion: "29939" uid: 7b099c88-39ff-44f4-9eb7-15191a3beb76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 2ba177c1-5524-4560-8e62-9ef0ce9f1cbf kind: Event lastTimestamp: "2026-04-23T14:13:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:13:39Z" name: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8.18a901e70ec2b020 namespace: kserve-ci-e2e-test resourceVersion: "29940" uid: 13f0a469-511e-4fbc-ad12-49cf5355d68b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a8c3e-predictor-965fc7978 namespace: kserve-ci-e2e-test resourceVersion: "25526" uid: 17e715a4-d412-4d94-8d0f-a0bef957b208 kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: 'Created pod: error-404-isvc-a8c3e-predictor-965fc7978-k4bn8' metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor-965fc7978.18a90165e5193c91 namespace: kserve-ci-e2e-test resourceVersion: "25535" uid: 32da5400-1bdc-444b-9523-f1f4b6b30f50 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25524" uid: fdd0659a-38bc-4fc9-a197-2c0ee9af1549 kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: Scaled up replica set error-404-isvc-a8c3e-predictor-965fc7978 from 0 to 1 metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e-predictor.18a90165e2f0e6b8 namespace: kserve-ci-e2e-test resourceVersion: "25528" uid: 3024263d-3c6c-400b-a791-92211dbedcfe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T14:04:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: 5f39eb36-ae8f-4bd8-8c1c-3b929d4054e1 kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:25Z" name: error-404-isvc-a8c3e-predictor.18a901696142ceaa namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 18754754-a9c0-4865-bd81-53d5d0f5052a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T14:04:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: 5f39eb36-ae8f-4bd8-8c1c-3b929d4054e1 kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:25Z" name: error-404-isvc-a8c3e-predictor.18a9016961b2a0ee namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: d01807d4-12f2-4952-beee-ff3d27c08f0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: 5f39eb36-ae8f-4bd8-8c1c-3b929d4054e1 kind: Event lastTimestamp: "2026-04-23T14:04:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:04:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:55Z" name: error-404-isvc-a8c3e-predictor.18a9016ce0813a5f namespace: kserve-ci-e2e-test resourceVersion: "25883" uid: c8068b13-eef4-471a-a88c-835705f727c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: 5f39eb36-ae8f-4bd8-8c1c-3b929d4054e1 kind: Event lastTimestamp: "2026-04-23T14:04:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:04:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:55Z" name: error-404-isvc-a8c3e-predictor.18a9016ce0fc1732 namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: 397978d0-6657-4cb5-b224-311023e9e484 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "25510" uid: c982846b-8d11-45b6-9392-800df6c8fa55 kind: Event lastTimestamp: "2026-04-23T14:05:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:05:13Z" name: error-404-isvc-a8c3e.18a90165e299f75a namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 422efb39-a162-4814-8b75-7b1ff88816d1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "25510" uid: c982846b-8d11-45b6-9392-800df6c8fa55 kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-a8c3e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a8c3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e.18a90165e32f0ef4 namespace: kserve-ci-e2e-test resourceVersion: "25530" uid: 49e5f515-965e-4356-8def-93025fd477f5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "25510" uid: c982846b-8d11-45b6-9392-800df6c8fa55 kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a8c3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:25Z" name: error-404-isvc-a8c3e.18a90165e32f610f namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: d3a61ef0-0f31-43ca-871e-43de51b583b8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "26326" uid: c982846b-8d11-45b6-9392-800df6c8fa55 kind: Event lastTimestamp: "2026-04-23T14:05:13Z" message: InferenceService [error-404-isvc-a8c3e] is Ready metadata: creationTimestamp: "2026-04-23T14:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:05:13Z" name: error-404-isvc-a8c3e.18a901711317db9b namespace: kserve-ci-e2e-test resourceVersion: "26327" uid: 18231065-23ce-442b-a78d-4c4d9f4d7154 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:43:34.617428Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:43:34Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a90042bd420ff0 namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: be217da9-787d-4797-a69b-8418b65004a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d2d70-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:34Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a90042c98cf553 namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: c68ecb1c-429f-4dad-9d1d-e564754bb1ef reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:43:35Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a90042f9c9a38d namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 75dc856f-3731-401d-8278-5d4c9e2ff882 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a90042fb37b274 namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 3e31dca6-0bd8-4a99-8e90-ce2c4b3467c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a90042fe54b62a namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: ce092897-316e-40b0-85f1-540929fe8f2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a90042feafc9bd namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: fdd70d16-2b5c-466b-809d-e64a097c949f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a90042febc009f namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 97d0215c-fe42-4f85-b468-9ff1342e5a60 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a9004301b46c35 namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: d2afafe7-4549-4382-98d3-e5322f1391a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a90043021064d0 namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: bede778d-5879-45a5-879a-4b075c9d4406 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:44:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:13Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a900436748940c namespace: kserve-ci-e2e-test resourceVersion: "15533" uid: af669e39-ae31-4e5f-9429-14de8d597804 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:49Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a900c3ed7d6aa5 namespace: kserve-ci-e2e-test resourceVersion: "19332" uid: aa7a36bc-33ed-43b5-bcd5-e562ff20b45c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: a96fbf6d-90ba-4223-868a-f95ba2c8d876 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:49Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk.18a900c3ed7dd602 namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: 187eb93a-7e80-4b02-bf3f-04b4e3d2ea16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d2d70-predictor-557fcdd5cb namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 981cd22d-c8e4-4888-8047-a7ca4d05b541 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: 'Created pod: error-404-isvc-d2d70-predictor-557fcdd5cb-lmqhk' metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:34Z" name: error-404-isvc-d2d70-predictor-557fcdd5cb.18a90042bcdd57f0 namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: 5208fa90-ddb1-424b-a02a-8ba7d0ee579d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: c0247d81-8f1a-4b5a-bbd7-f564ec99beb6 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Scaled up replica set error-404-isvc-d2d70-predictor-557fcdd5cb from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:34Z" name: error-404-isvc-d2d70-predictor.18a90042bb4e3020 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 737d146e-5229-4fc4-84ea-a95860f5c5f0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:43:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: 5f19697a-2fc3-459b-ab0e-5c479c939edd kind: Event lastTimestamp: "2026-04-23T13:44:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:04Z" name: error-404-isvc-d2d70-predictor.18a9004639bed319 namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 2078eeba-286d-4210-8d8b-df2ec23e9bd4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:43:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: 5f19697a-2fc3-459b-ab0e-5c479c939edd kind: Event lastTimestamp: "2026-04-23T13:44:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:04Z" name: error-404-isvc-d2d70-predictor.18a900463a133749 namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: 62a210b6-26ac-4d01-82c4-4c493bc48380 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 5f19697a-2fc3-459b-ab0e-5c479c939edd kind: Event lastTimestamp: "2026-04-23T13:44:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:49Z" name: error-404-isvc-d2d70-predictor.18a9004d37640ec1 namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 3bb13b05-fb13-4b15-b35c-8a167c71b052 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 5f19697a-2fc3-459b-ab0e-5c479c939edd kind: Event lastTimestamp: "2026-04-23T13:44:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:49Z" name: error-404-isvc-d2d70-predictor.18a9004d37c7eb2e namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 4466a0be-f335-4f9c-bc08-6d309bc74f18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: ad01d848-b1ec-4b71-a0f9-bd3b40337d08 kind: Event lastTimestamp: "2026-04-23T13:44:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:23Z" name: error-404-isvc-d2d70.18a90042bb2a0f27 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: e8523b07-d3a1-4009-8d66-45f69a544ebf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: ad01d848-b1ec-4b71-a0f9-bd3b40337d08 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: 'Failed to update status for InferenceService "error-404-isvc-d2d70": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d2d70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:34Z" name: error-404-isvc-d2d70.18a90042bb7bf3a3 namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: 845c1e4f-fd6a-46cd-bbe7-1fbffd862033 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: ad01d848-b1ec-4b71-a0f9-bd3b40337d08 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d2d70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:34Z" name: error-404-isvc-d2d70.18a90042bb7c4777 namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: 77d64b96-6849-43d7-8ca8-38c54e246cc3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: ad01d848-b1ec-4b71-a0f9-bd3b40337d08 kind: Event lastTimestamp: "2026-04-23T13:44:23Z" message: InferenceService [error-404-isvc-d2d70] is Ready metadata: creationTimestamp: "2026-04-23T13:44:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:23Z" name: error-404-isvc-d2d70.18a9004e21eece98 namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: ccb1ac84-edde-4b6d-b5b9-4ea555daad0a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:44:10.483854Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f7bdd-predictor-84546c4756-jjbxj to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a9004b17104a56 namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 8a7e6100-4d34-47c4-a7bc-20243e205e40 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a9004b2fe89568 namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: 93ac2b1d-c4b2-4a9b-b56a-6660779d3bf8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a9004b31337517 namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: c64dc083-0362-447a-ac17-fe9128699b48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a9004b340d3b7c namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 872dc415-6406-49f9-8cf9-a6ad1e4b0497 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a9004b34807892 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: 48bd8866-824a-4c7c-a036-1f179196b558 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a9004b348c610e namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: bfd41ec4-091f-4353-b02e-cccc568cb589 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:44:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:11Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a9004b377299ce namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: 5b65d225-bd91-4a13-a946-03234ed27c9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:44:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:11Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a9004b37de4adb namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 58a58b46-fce9-45af-bcfd-0af526789db2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:44:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:44:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:44:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:48Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a9004b93a56ff4 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 25330312-27ea-43a3-b7c9-72c941735480 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a900cc451758f7 namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: d1ff81bc-e632-4dd5-89fb-30a2c099ea81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9586a1e0-fab0-42ee-8108-aca7842dcef2 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj.18a900cc4517b95a namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: ed082d43-2583-4a7b-b942-53a469618ccd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f7bdd-predictor-84546c4756 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 7c5024db-00e9-4110-aa5b-2f2eb517f831 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: 'Created pod: error-404-isvc-f7bdd-predictor-84546c4756-jjbxj' metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd-predictor-84546c4756.18a9004b167c4325 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 331cc139-ff0f-4c3f-a0c0-a9aff2156578 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: 52f81c67-ffd1-4561-8f9b-d20368b6dfbf kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Scaled up replica set error-404-isvc-f7bdd-predictor-84546c4756 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd-predictor.18a9004b151834e7 namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: f7950014-6552-4214-9a81-5a28ef14ac8a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: c61faef5-5126-49c3-a339-5c295884d616 kind: Event lastTimestamp: "2026-04-23T13:44:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:40Z" name: error-404-isvc-f7bdd-predictor.18a9004e93eff3a1 namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: b6fe79d2-5b5d-4283-8ca9-15341e239bfc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: c61faef5-5126-49c3-a339-5c295884d616 kind: Event lastTimestamp: "2026-04-23T13:44:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:40Z" name: error-404-isvc-f7bdd-predictor.18a9004e9455b465 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 526fc9dc-3013-4453-b84e-39b57f41c69f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:44:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: c61faef5-5126-49c3-a339-5c295884d616 kind: Event lastTimestamp: "2026-04-23T13:45:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:10Z" name: error-404-isvc-f7bdd-predictor.18a9005591d734cf namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: d46d562e-70d6-4763-b17b-1bebe446131b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:44:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: c61faef5-5126-49c3-a339-5c295884d616 kind: Event lastTimestamp: "2026-04-23T13:45:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:10Z" name: error-404-isvc-f7bdd-predictor.18a90055923762f6 namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 7c72affa-d5aa-4ff1-ac36-047d41f15c1d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: b8ed2c41-fa5f-444d-8886-7f8093f7118e kind: Event lastTimestamp: "2026-04-23T13:44:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:58Z" name: error-404-isvc-f7bdd.18a9004b154dd4eb namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: f666f2a6-7b96-435a-bdd8-b3b4e5aeb79b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: b8ed2c41-fa5f-444d-8886-7f8093f7118e kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-f7bdd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7bdd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd.18a9004b159a548f namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: baffd7d1-d7d5-4014-8b4d-2dea02aac077 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: b8ed2c41-fa5f-444d-8886-7f8093f7118e kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7bdd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:10Z" name: error-404-isvc-f7bdd.18a9004b159a84f9 namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: c2ae5512-b52c-4277-a9a2-1e7cd23ace93 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: b8ed2c41-fa5f-444d-8886-7f8093f7118e kind: Event lastTimestamp: "2026-04-23T13:44:58Z" message: InferenceService [error-404-isvc-f7bdd] is Ready metadata: creationTimestamp: "2026-04-23T13:44:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:58Z" name: error-404-isvc-f7bdd.18a900564eac0250 namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: ca10d6be-b0a5-479b-8d3f-86a88f7ab37e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T14:03:50.599895Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T14:03:50Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9015ddb7612c9 namespace: kserve-ci-e2e-test resourceVersion: "25060" uid: 13d40fe6-910b-45c1-9912-a8a1dc70b9ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25064" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:03:51Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T14:03:51Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9015df4bd7193 namespace: kserve-ci-e2e-test resourceVersion: "25080" uid: 260112ed-0b80-4f07-a314-a9742fe4d3bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:03:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:51Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9015df642a6fe namespace: kserve-ci-e2e-test resourceVersion: "25082" uid: 26b0beb7-9ab5-48d8-96d8-9bb682813c14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:03:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:51Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9015df928efa8 namespace: kserve-ci-e2e-test resourceVersion: "25083" uid: 8f61ebe7-8ad0-4a54-87bc-5acf39621fe4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:03:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:51Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9015df9ad579b namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: bd72f259-b3f7-4cf8-8cff-70cda9832ca3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:03:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:51Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9015df9b82014 namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: fa980c20-7971-4b11-9c73-2aa7f058e293 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:03:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:51Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9015dfcac7344 namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: 2f617f6e-f174-453b-9e31-8123b8334b56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:03:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:51Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9015dfd19c814 namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: a7ba0e41-a6cb-4d17-a116-95e78bb8de6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:04:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T14:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:28Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9015e667eb839 namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: e05a2576-1660-4094-8e82-0359dc14b44c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:21:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:21:10Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9024fdca21d20 namespace: kserve-ci-e2e-test resourceVersion: "33077" uid: 62e9c649-3a02-401e-a215-1c619fb86d4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 7830be0b-2892-4e47-b17d-bb0aa34efc73 kind: Event lastTimestamp: "2026-04-23T14:21:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:21:10Z" name: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk.18a9024fdca35139 namespace: kserve-ci-e2e-test resourceVersion: "33078" uid: 5cfdc176-1737-4f91-aba7-f751bddd28c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-feffa-predictor-75cd96c4d6 namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: df0bdbef-c6bb-4666-9f0f-655c85284716 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: 'Created pod: error-404-isvc-feffa-predictor-75cd96c4d6-tzwhk' metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:50Z" name: error-404-isvc-feffa-predictor-75cd96c4d6.18a9015ddb2003ca namespace: kserve-ci-e2e-test resourceVersion: "25058" uid: ef6aed32-dbe1-4944-991c-bf5b741f7520 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25041" uid: 344d1640-e8f4-4d76-9d26-305e30cd07c0 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Scaled up replica set error-404-isvc-feffa-predictor-75cd96c4d6 from 0 to 1 metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:50Z" name: error-404-isvc-feffa-predictor.18a9015dd800efd9 namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: d0c66da4-d170-484d-8a6a-08f281dfde12 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:04:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25042" uid: 69fe7d0b-0112-451f-84d1-11d186cbbd96 kind: Event lastTimestamp: "2026-04-23T14:04:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:20Z" name: error-404-isvc-feffa-predictor.18a9016156968162 namespace: kserve-ci-e2e-test resourceVersion: "25400" uid: 3fe5877a-2b87-4ef6-b684-b98cafe722f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:04:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25042" uid: 69fe7d0b-0112-451f-84d1-11d186cbbd96 kind: Event lastTimestamp: "2026-04-23T14:04:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:20Z" name: error-404-isvc-feffa-predictor.18a901615701e006 namespace: kserve-ci-e2e-test resourceVersion: "25401" uid: 395c2a75-e1a1-4f95-b0aa-327f0f184fe5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T14:04:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25241" uid: 69fe7d0b-0112-451f-84d1-11d186cbbd96 kind: Event lastTimestamp: "2026-04-23T14:05:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:05Z" name: error-404-isvc-feffa-predictor.18a901685479151c namespace: kserve-ci-e2e-test resourceVersion: "26149" uid: 7e9ca3d5-9e51-414c-8c3a-82b80d452e42 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T14:04:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25241" uid: 69fe7d0b-0112-451f-84d1-11d186cbbd96 kind: Event lastTimestamp: "2026-04-23T14:05:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:05Z" name: error-404-isvc-feffa-predictor.18a9016854e32bbd namespace: kserve-ci-e2e-test resourceVersion: "26150" uid: 466dade2-f3b2-494d-bfd5-6ab16850df90 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-feffa namespace: kserve-ci-e2e-test resourceVersion: "25013" uid: e147c52a-f30c-4ff5-a77a-d1c32bb4c98e kind: Event lastTimestamp: "2026-04-23T14:04:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:39Z" name: error-404-isvc-feffa.18a9015dd7ea7e9f namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: ecfe618c-a1d4-4ed6-ad57-d8dbe3833eda reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-feffa namespace: kserve-ci-e2e-test resourceVersion: "25013" uid: e147c52a-f30c-4ff5-a77a-d1c32bb4c98e kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-feffa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-feffa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:03:50Z" name: error-404-isvc-feffa.18a9015dd8aacc4a namespace: kserve-ci-e2e-test resourceVersion: "25048" uid: 57860ad2-3cca-4b6b-86d0-a7a120656e48 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-feffa namespace: kserve-ci-e2e-test resourceVersion: "25013" uid: e147c52a-f30c-4ff5-a77a-d1c32bb4c98e kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-feffa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:03:50Z" name: error-404-isvc-feffa.18a9015dd8ab17f9 namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: 228ca6c6-7532-4844-983f-ed10a1303c45 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-feffa namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: e147c52a-f30c-4ff5-a77a-d1c32bb4c98e kind: Event lastTimestamp: "2026-04-23T14:04:39Z" message: InferenceService [error-404-isvc-feffa] is Ready metadata: creationTimestamp: "2026-04-23T14:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:39Z" name: error-404-isvc-feffa.18a9016921cba95d namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: 422d0102-bb34-4f4b-b2d6-5c5d1591ca7b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:42:00.286182Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:42:00Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9002cc6ad76b3 namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: 1a7c5d7d-3f45-496e-b8e6-6cb93e288bc8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:01Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9002cd3b56826 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: bb20baa7-49e1-4fa6-b8d7-94c21cc0853f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13817" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:02Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:42:02Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9002d3b0843ce namespace: kserve-ci-e2e-test resourceVersion: "13929" uid: 9537d3f7-cfe2-43dc-a679-3677452e1413 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:02Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" metadata: creationTimestamp: "2026-04-23T13:42:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:02Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9002d3c97a078 namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: df3f7f72-2180-47a8-8fe3-9c75c0c11b77 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" in 11.996s (11.996s including waiting). Image size: 301492121 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:14Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9003007a4e97b namespace: kserve-ci-e2e-test resourceVersion: "14049" uid: a5078517-6506-4ef1-8b8b-9f1fd675c7b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:14Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a900300b84b926 namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 578b6b01-bf24-4333-a304-6a3446ce0c45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:14Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a900300be27804 namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: 86b46470-f804-4560-8df1-585bc03c41b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:18Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" metadata: creationTimestamp: "2026-04-23T13:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:18Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a90030f4e6a823 namespace: kserve-ci-e2e-test resourceVersion: "14132" uid: 872b6aad-cd45-45d7-bf3b-a6e6e3d42aa8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:24Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" in 6.167s (6.168s including waiting). Image size: 1562812843 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:24Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a90032648b4f27 namespace: kserve-ci-e2e-test resourceVersion: "14187" uid: 9cd10491-c537-4ffa-9052-e970f43c8418 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:24Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9003267569a44 namespace: kserve-ci-e2e-test resourceVersion: "14188" uid: 4e0d9422-a4e7-4d04-b3d6-82b69dcad0fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:24Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9003267bc07bc namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 02a672b0-e47d-445a-bf73-fac56fc19e77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:24Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9003267d32ed5 namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: eca64d55-b725-42f0-bec4-b016030427d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:24Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a900326ad28079 namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 04fb0d9f-82f0-438e-85be-16b446221510 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:42:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:24Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a900326b32b32e namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 9622dc04-8a68-4257-8168-f082d3bd18d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T13:42:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:44:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:12Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a90032d33ca10a namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 0ec3663d-7a2a-41f5-bd5c-8ef501549923 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9004b1048cdbb namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: 7dfb454f-23d3-43bb-b4ce-17f9b2de7478 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9004b104932ea namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 28707afb-e603-428f-ab52-37275bf424a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8 namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 01b5a2d1-4b2c-4634-a26c-0eb0f0d31d1e kind: Event lastTimestamp: "2026-04-23T13:44:12Z" message: 'Readiness probe failed: Get "https://10.132.0.17:8643/healthz": dial tcp 10.132.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:44:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:12Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8.18a9004b816cf5ea namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 47b9117b-e93f-4fca-b4d4-b32359f330b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7c88655ddf namespace: kserve-ci-e2e-test resourceVersion: "13796" uid: d14361a1-db78-429e-a2bc-3c30f9f8f2c0 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7c88655ddf-p6px8' metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:00Z" name: isvc-sklearn-graph-1-predictor-7c88655ddf.18a9002cc639d780 namespace: kserve-ci-e2e-test resourceVersion: "13806" uid: fd7576fd-9b15-483a-98a5-70a270601565 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: fb7d0d3a-3e77-4bfe-b009-6f6d9c6232e2 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7c88655ddf from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:00Z" name: isvc-sklearn-graph-1-predictor.18a9002cc393f8c6 namespace: kserve-ci-e2e-test resourceVersion: "13799" uid: 1e67c506-a2c3-427d-98bc-85772f7553d2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13795" uid: 2d7b9b3f-095b-4ecc-82b0-c1598dce7abf kind: Event lastTimestamp: "2026-04-23T13:43:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:00Z" name: isvc-sklearn-graph-1-predictor.18a9003042170338 namespace: kserve-ci-e2e-test resourceVersion: "14505" uid: c553b26c-77f7-43d6-8851-1ded92d31648 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13795" uid: 2d7b9b3f-095b-4ecc-82b0-c1598dce7abf kind: Event lastTimestamp: "2026-04-23T13:43:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:00Z" name: isvc-sklearn-graph-1-predictor.18a9003042e98bf4 namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: f9b4af50-f612-4211-9875-ad9667e0cacf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:42:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14070" uid: 2d7b9b3f-095b-4ecc-82b0-c1598dce7abf kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:00Z" name: isvc-sklearn-graph-1-predictor.18a900374040e89f namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: 0a19e555-1f41-4537-a62e-177a811dad92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:42:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14070" uid: 2d7b9b3f-095b-4ecc-82b0-c1598dce7abf kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:00Z" name: isvc-sklearn-graph-1-predictor.18a9003740a02a8f namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 7bec2bb8-8efe-40b8-b60e-3cd6324f70d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13779" uid: 6f9642ab-68b5-45a6-bacc-3c0252635985 kind: Event lastTimestamp: "2026-04-23T13:43:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:32Z" name: isvc-sklearn-graph-1.18a9002cc36eec09 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 4fdec782-97ac-4c47-b814-672196625a82 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: 6f9642ab-68b5-45a6-bacc-3c0252635985 kind: Event lastTimestamp: "2026-04-23T13:43:32Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-23T13:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:32Z" name: isvc-sklearn-graph-1.18a90042365687f2 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 39e07523-ebcb-48df-975a-95869c71ea03 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:42:01.239851Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13902" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-565675b447-t5g7p to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T13:42:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:42:01Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a9002cff8541db namespace: kserve-ci-e2e-test resourceVersion: "13906" uid: be510da2-707f-45a8-9e44-c594c1d2b8b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13910" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:01Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:42:01Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a9002d1ade6820 namespace: kserve-ci-e2e-test resourceVersion: "13917" uid: 54a414e1-6585-4aa3-a127-c32151db1199 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:01Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" metadata: creationTimestamp: "2026-04-23T13:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:01Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a9002d1c7d4337 namespace: kserve-ci-e2e-test resourceVersion: "13921" uid: 1e204e09-320a-417a-b981-bc652cffc9b7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" in 13.609s (13.609s including waiting). Image size: 301492121 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:15Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a9003047b1caae namespace: kserve-ci-e2e-test resourceVersion: "14076" uid: 992648c1-d878-4704-bdf9-148ed9bcdb78 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:15Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a900304b335fa9 namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: a3f80449-a02a-47e4-9ab9-56ffb9d76a04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:15Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a900304bbcad1b namespace: kserve-ci-e2e-test resourceVersion: "14083" uid: 63cf055c-b468-4779-a435-27d7d94d83d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:19Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" metadata: creationTimestamp: "2026-04-23T13:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:20Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a900315dad9489 namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: 9aca6545-1e8c-4b9a-9879-dc0119d302bc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:25Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" in 5.743s (5.743s including waiting). Image size: 1562812843 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:25Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a90032b40280db namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: b6196e4c-fca0-47cf-9d66-1d44d7b3a200 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:25Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a90032b825ae3a namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: 2dbdb615-4156-4d72-be98-171cc6e5440a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:25Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a90032b89e14ad namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: 95160885-e907-4d7d-8796-513556e24872 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:25Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a90032b8b66582 namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: e617bfb1-3c56-4655-bab4-9ee45f485f0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:25Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a90032bbbde1d5 namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 1fd48296-659e-45f0-81a0-0ba71fef8739 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:42:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:25Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a90032bc308fcd namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: 2a25a672-d4a2-4415-845e-a2806d73fc48 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:44:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:13Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a900330078a80b namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 42049a67-4c1b-4c6a-87ea-82e229ad880c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a9004b0cfbe45e namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: eeb54bea-92bc-4029-a5a1-5c5a7821800d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a9004b0cfc636f namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: 914280b8-2a20-40fa-8181-fd3587eb01d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: a6d11ee3-1e59-43fa-8a0e-20c272188933 kind: Event lastTimestamp: "2026-04-23T13:44:13Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:13Z" name: isvc-sklearn-graph-2-predictor-565675b447-t5g7p.18a9004baeae1371 namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: f5e97958-7f4e-4704-839c-11e68236c1ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-565675b447 namespace: kserve-ci-e2e-test resourceVersion: "13886" uid: 7dfbcffc-eda1-43d0-b4ea-a111c4dff4d2 kind: Event lastTimestamp: "2026-04-23T13:42:01Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-565675b447-t5g7p' metadata: creationTimestamp: "2026-04-23T13:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:01Z" name: isvc-sklearn-graph-2-predictor-565675b447.18a9002cff10b383 namespace: kserve-ci-e2e-test resourceVersion: "13903" uid: 4efbe4a7-e41d-4e54-b82d-d325932c37f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13884" uid: 23e8bab1-50e4-4123-852e-8c24079a3e2a kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-565675b447 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:00Z" name: isvc-sklearn-graph-2-predictor.18a9002ce8b0471d namespace: kserve-ci-e2e-test resourceVersion: "13887" uid: fc793238-a4ae-4b45-a033-acaabd8a371a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13885" uid: 18a125b1-6ee9-4913-8756-c232bbccc849 kind: Event lastTimestamp: "2026-04-23T13:43:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:00Z" name: isvc-sklearn-graph-2-predictor.18a900306714e2de namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: 22cd7c44-a8ed-44fb-8d86-a2360ebc3d0a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13885" uid: 18a125b1-6ee9-4913-8756-c232bbccc849 kind: Event lastTimestamp: "2026-04-23T13:43:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:00Z" name: isvc-sklearn-graph-2-predictor.18a9003067760a65 namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: ac003080-9e3c-4664-af6e-d7a06ac6925c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:42:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 18a125b1-6ee9-4913-8756-c232bbccc849 kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:00Z" name: isvc-sklearn-graph-2-predictor.18a9003764fd851f namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 746be567-c0a2-4053-97d5-0dcc8162ccd0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:42:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 18a125b1-6ee9-4913-8756-c232bbccc849 kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:00Z" name: isvc-sklearn-graph-2-predictor.18a900376564795b namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: 90b14823-d238-4b06-b6bf-2e908838dbf6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13861" uid: 2a41f138-3300-460f-a136-0b685b24854a kind: Event lastTimestamp: "2026-04-23T13:43:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:33Z" name: isvc-sklearn-graph-2.18a9002ce883a6a1 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d1c991bf-1d42-43c9-b57d-f80f56f7031d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 2a41f138-3300-460f-a136-0b685b24854a kind: Event lastTimestamp: "2026-04-23T13:43:33Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-23T13:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:33Z" name: isvc-sklearn-graph-2.18a9004263d30005 namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: c4c96fd2-8926-4fc8-aaf6-0afe7502a728 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:42:00.383559Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13830" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-kk2qw to ip-10-0-137-177.ec2.internal metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:42:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9002ccc7b57fd namespace: kserve-ci-e2e-test resourceVersion: "13834" uid: 75bb46b0-de40-4d92-b026-aec0407e481d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13842" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:42:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9002ce5ee5382 namespace: kserve-ci-e2e-test resourceVersion: "13875" uid: 0d7f9bc8-32ec-454f-9b6e-a67970ce7bbf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9002ce74b96c4 namespace: kserve-ci-e2e-test resourceVersion: "13881" uid: 4680b4dd-ef4d-49c4-866e-8baa3f51c431 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" in 3.537s (3.537s including waiting). Image size: 301492121 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:04Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9002dba258a9d namespace: kserve-ci-e2e-test resourceVersion: "13959" uid: 429cd57e-fedd-4723-b3d9-e7d6417c35e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:04Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9002dbe4e0609 namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: 6ca498fa-6bfd-40ab-9537-f3ac6b3de879 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:04Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9002dbec6da61 namespace: kserve-ci-e2e-test resourceVersion: "13963" uid: 1ee8f107-19a7-4350-bd14-0150fbdeb67d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:09Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-23T13:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:09Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9002ed732f51e namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: 25435497-8999-479e-8832-f69704d72d29 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.455s (18.455s including waiting). Image size: 1306414326 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9003323376e48 namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 4de3b8d0-e3ac-4357-a65e-5bee5216c74d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a900332804fc1b namespace: kserve-ci-e2e-test resourceVersion: "14228" uid: 272b9bea-1b08-4498-b825-3fc8037c682c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9003328eb0248 namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: d1e5b740-0eb8-49d2-b97d-d7919cbdc5ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9003328f78823 namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: 80b59e96-6f22-4dec-a2f5-29bf5c29657f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:30Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.375s (2.375s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a90033b68f90a6 namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 1d27d97b-364d-415a-9589-754f13b8191f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a90033ba3f8f56 namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: aa57883a-2b53-483b-ab17-c120c96f376e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:42:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a90033bab6bf25 namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: a6447a98-766c-4ca0-8c5d-7e3536a87328 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T13:42:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:43:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a90033fbf591a7 namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: b368c717-574c-4d49-b3cc-02aa9ab1b951 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9004b14838f2b namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 8e0fb920-f5c3-4bcb-832c-b9626427e60d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9004b148622f2 namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 284ec3d6-b52c-4c89-9c4b-00190f3753f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 6037a5f7-804d-40a1-8f02-adb635079917 kind: Event lastTimestamp: "2026-04-23T13:44:12Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:44:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:12Z" name: isvc-xgboost-graph-predictor-669d8d6456-kk2qw.18a9004b802d4f4d namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: d866fc70-524b-4503-bb45-b69e2c5b2995 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-177.ec2.internal source: component: kubelet host: ip-10-0-137-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "13822" uid: f6533a4e-1139-4391-830d-2a5db1841346 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-kk2qw' metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:00Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a9002ccbf2c5ea namespace: kserve-ci-e2e-test resourceVersion: "13831" uid: f1b464ce-4991-4f49-b24f-2db582e25867 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13820" uid: f93ce52b-84d0-486e-8dd8-96ec3904f2d6 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:00Z" name: isvc-xgboost-graph-predictor.18a9002cc94084a6 namespace: kserve-ci-e2e-test resourceVersion: "13824" uid: c9c8af11-5247-435a-ad6c-0275a64d05e6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13821" uid: 2f0f1748-1e6c-4172-83c8-f20ac9212d78 kind: Event lastTimestamp: "2026-04-23T13:43:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:00Z" name: isvc-xgboost-graph-predictor.18a9003047f0a475 namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: bbed3f98-cede-4a1d-8d9b-37c761b14ce0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13821" uid: 2f0f1748-1e6c-4172-83c8-f20ac9212d78 kind: Event lastTimestamp: "2026-04-23T13:43:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:00Z" name: isvc-xgboost-graph-predictor.18a9003048235ce6 namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: 36214ba5-b01f-4d2a-add7-726f6d02c0af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:43:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: 2f0f1748-1e6c-4172-83c8-f20ac9212d78 kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:00Z" name: isvc-xgboost-graph-predictor.18a9003e42cf8bab namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: a915ae72-6cb4-436a-8fc1-3969298869df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:43:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: 2f0f1748-1e6c-4172-83c8-f20ac9212d78 kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:00Z" name: isvc-xgboost-graph-predictor.18a9003e432fcc89 namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: 77d505ff-74d1-4d3a-a6ee-b1269846dcbb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13803" uid: 4a1dcfb4-3d51-47da-a966-cb67af5f08bc kind: Event lastTimestamp: "2026-04-23T13:43:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:37Z" name: isvc-xgboost-graph.18a9002cc91fcc71 namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 84666d00-42f4-4d8f-bb10-9da652d1a85d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 4a1dcfb4-3d51-47da-a966-cb67af5f08bc kind: Event lastTimestamp: "2026-04-23T13:43:37Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-23T13:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:37Z" name: isvc-xgboost-graph.18a900435f742b46 namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: 8dcfa778-b330-4585-837e-e00617122410 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:44:00.123542Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5649869f87-hrjlf namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 04bceceb-c66a-4738-bf8f-ea4f233212cc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5649869f87-hrjlf to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:44:00Z" name: model-chainer-5649869f87-hrjlf.18a90048ad8a7f62 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 86b86b88-ae58-4da7-846b-dd5a6396615a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:00Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5649869f87-hrjlf namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: 04bceceb-c66a-4738-bf8f-ea4f233212cc kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:44:00Z" name: model-chainer-5649869f87-hrjlf.18a90048c64be1b2 namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: 0a22a0ad-d745-47da-b871-6993faf2be85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5649869f87-hrjlf namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 04bceceb-c66a-4738-bf8f-ea4f233212cc kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:00Z" name: model-chainer-5649869f87-hrjlf.18a90048c7949ba3 namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 5445c565-a1f1-4dc5-a5d5-3c2bb4e9fc58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5649869f87-hrjlf namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 04bceceb-c66a-4738-bf8f-ea4f233212cc kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:00Z" name: model-chainer-5649869f87-hrjlf.18a90048ca71d12f namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: b498f222-1af1-4b6c-9063-c893d14c9e4b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5649869f87-hrjlf namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 04bceceb-c66a-4738-bf8f-ea4f233212cc kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:00Z" name: model-chainer-5649869f87-hrjlf.18a90048cae0407b namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 0434321b-34b7-4f27-8ec4-1f50a903ead5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5649869f87-hrjlf namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 04bceceb-c66a-4738-bf8f-ea4f233212cc kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: model-chainer-5649869f87-hrjlf.18a9004b07de5700 namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: 74804281-7f83-45c5-92ac-640e53c51541 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:44:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5649869f87-hrjlf namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 04bceceb-c66a-4738-bf8f-ea4f233212cc kind: Event lastTimestamp: "2026-04-23T13:44:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:44:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:37Z" name: model-chainer-5649869f87-hrjlf.18a9004b91c8fd2f namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: dad871ee-7e26-4082-b05d-ed5a7ea0d3cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5649869f87 namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: d5acd6fd-ec32-43eb-a22c-0beade67dd4c kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: 'Created pod: model-chainer-5649869f87-hrjlf' metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:00Z" name: model-chainer-5649869f87.18a90048ad1b88bd namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: d979aa48-a3b7-4d76-ac5d-c372376b4375 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: cec74671-f04f-4f46-aa60-e1c5cef548c0 kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: Scaled up replica set model-chainer-5649869f87 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:00Z" name: model-chainer.18a90048ac402c22 namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: e634f38b-2bd0-4c7f-8f9a-2a5812fb76fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: ba7904ef-83f5-4e8e-8989-845cabdbc419 kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:00Z" name: model-chainer.18a90048b28a243c namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: b0ca5f4b-6f2f-41fd-9937-b79d89c062a2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:54:25.322629Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1cfcc-57cb468459-llds7 namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: 10263db1-913b-4de1-b858-7eccae24824d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-1cfcc-57cb468459-llds7 to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:54:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:54:25Z" name: sequence-graph-1cfcc-57cb468459-llds7.18a900da3e4f6d01 namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 2a2c1e2c-a917-4e9a-a8b9-11b6a0d0c85b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:25Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1cfcc-57cb468459-llds7 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 10263db1-913b-4de1-b858-7eccae24824d kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:54:25Z" name: sequence-graph-1cfcc-57cb468459-llds7.18a900da56eaa380 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: f93aea01-caad-4fab-9eb1-2cbab56fb006 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1cfcc} kind: Pod name: sequence-graph-1cfcc-57cb468459-llds7 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 10263db1-913b-4de1-b858-7eccae24824d kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T13:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:25Z" name: sequence-graph-1cfcc-57cb468459-llds7.18a900da58fcf9cd namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: e0638a43-c8d2-4cc1-81e1-a6d8106e046e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1cfcc} kind: Pod name: sequence-graph-1cfcc-57cb468459-llds7 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 10263db1-913b-4de1-b858-7eccae24824d kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: 'Created container: sequence-graph-1cfcc' metadata: creationTimestamp: "2026-04-23T13:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:25Z" name: sequence-graph-1cfcc-57cb468459-llds7.18a900da5c0daaf8 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 899b073e-567d-40cf-a0a3-0693f1b60b10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1cfcc} kind: Pod name: sequence-graph-1cfcc-57cb468459-llds7 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 10263db1-913b-4de1-b858-7eccae24824d kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: Started container sequence-graph-1cfcc metadata: creationTimestamp: "2026-04-23T13:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:25Z" name: sequence-graph-1cfcc-57cb468459-llds7.18a900da5c8fa9d3 namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: 164bdeee-8394-4952-8adb-b162503bf5fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1cfcc} kind: Pod name: sequence-graph-1cfcc-57cb468459-llds7 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 10263db1-913b-4de1-b858-7eccae24824d kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: Stopping container sequence-graph-1cfcc metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:35Z" name: sequence-graph-1cfcc-57cb468459-llds7.18a900dc9436427a namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: f638db48-104b-4191-98a7-5436943b55d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:54:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1cfcc} kind: Pod name: sequence-graph-1cfcc-57cb468459-llds7 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 10263db1-913b-4de1-b858-7eccae24824d kind: Event lastTimestamp: "2026-04-23T13:55:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:54:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:55:02Z" name: sequence-graph-1cfcc-57cb468459-llds7.18a900dd09bb10fa namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: 80d09747-a3c9-4654-a7bd-8cf843e5c766 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-1cfcc-57cb468459 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: cae80be4-7fe1-4c0a-9847-d735a5e3eb00 kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: 'Created pod: sequence-graph-1cfcc-57cb468459-llds7' metadata: creationTimestamp: "2026-04-23T13:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:25Z" name: sequence-graph-1cfcc-57cb468459.18a900da3dfc63b9 namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: b195ae8a-66f6-4f09-a59b-c41bd2d8e717 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 3effa997-dfbe-46d9-8249-f1acdcaea498 kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: Scaled up replica set sequence-graph-1cfcc-57cb468459 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:25Z" name: sequence-graph-1cfcc.18a900da3c6ef0ad namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: d5303b56-2e59-4d69-9366-b62132156743 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 1bd77444-c60c-4a51-955b-62f728cf8d52 kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: InferenceGraph [sequence-graph-1cfcc] is Ready metadata: creationTimestamp: "2026-04-23T13:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:25Z" name: sequence-graph-1cfcc.18a900da47330a0f namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 36bf2a0b-61e2-4c9f-bc71-61d627fff69b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:55:35.572859Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d150-55944bff46-k6lxc namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: e92d4441-cbca-43ee-9969-2d4880ddff10 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8d150-55944bff46-k6lxc to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:55:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:55:35Z" name: sequence-graph-8d150-55944bff46-k6lxc.18a900ea998ceb76 namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: b51aa843-4ae2-433e-b5e1-a1311209c0bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:35Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d150-55944bff46-k6lxc namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: e92d4441-cbca-43ee-9969-2d4880ddff10 kind: Event lastTimestamp: "2026-04-23T13:55:35Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:55:35Z" name: sequence-graph-8d150-55944bff46-k6lxc.18a900eab2404697 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 88663973-ff32-43a0-b4e5-877949f30c24 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d150} kind: Pod name: sequence-graph-8d150-55944bff46-k6lxc namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: e92d4441-cbca-43ee-9969-2d4880ddff10 kind: Event lastTimestamp: "2026-04-23T13:55:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T13:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:55:36Z" name: sequence-graph-8d150-55944bff46-k6lxc.18a900eab3b9c40c namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: f5ad4730-c108-4f20-9858-49876b106268 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d150} kind: Pod name: sequence-graph-8d150-55944bff46-k6lxc namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: e92d4441-cbca-43ee-9969-2d4880ddff10 kind: Event lastTimestamp: "2026-04-23T13:55:36Z" message: 'Created container: sequence-graph-8d150' metadata: creationTimestamp: "2026-04-23T13:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:55:36Z" name: sequence-graph-8d150-55944bff46-k6lxc.18a900eab65a25f1 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: 98ff04c3-c1fe-4487-9cc6-5586dfd06b1e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d150} kind: Pod name: sequence-graph-8d150-55944bff46-k6lxc namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: e92d4441-cbca-43ee-9969-2d4880ddff10 kind: Event lastTimestamp: "2026-04-23T13:55:36Z" message: Started container sequence-graph-8d150 metadata: creationTimestamp: "2026-04-23T13:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:55:36Z" name: sequence-graph-8d150-55944bff46-k6lxc.18a900eab6c6fa8e namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 218315dc-e929-4ef7-9dc3-4029f04a319f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d150} kind: Pod name: sequence-graph-8d150-55944bff46-k6lxc namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: e92d4441-cbca-43ee-9969-2d4880ddff10 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Stopping container sequence-graph-8d150 metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:50Z" name: sequence-graph-8d150-55944bff46-k6lxc.18a9015dc5a02a81 namespace: kserve-ci-e2e-test resourceVersion: "24974" uid: 275b6b3d-cbd8-42d8-adcf-278c43ae2295 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d150} kind: Pod name: sequence-graph-8d150-55944bff46-k6lxc namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: e92d4441-cbca-43ee-9969-2d4880ddff10 kind: Event lastTimestamp: "2026-04-23T14:04:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T14:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:17Z" name: sequence-graph-8d150-55944bff46-k6lxc.18a9015e4f438ce2 namespace: kserve-ci-e2e-test resourceVersion: "25372" uid: 5bfa2af0-a5b7-47c8-bed3-890a641906cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8d150-55944bff46 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 37780a32-d8ce-4dd5-b420-aff13af23814 kind: Event lastTimestamp: "2026-04-23T13:55:35Z" message: 'Created pod: sequence-graph-8d150-55944bff46-k6lxc' metadata: creationTimestamp: "2026-04-23T13:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:35Z" name: sequence-graph-8d150-55944bff46.18a900ea9922e068 namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: 2a3e5905-8a06-42e3-8cf2-d8616a374f6f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8d150 namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 8c580b75-a118-4cb2-b93c-6eec963aa38b kind: Event lastTimestamp: "2026-04-23T13:55:35Z" message: Scaled up replica set sequence-graph-8d150-55944bff46 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:35Z" name: sequence-graph-8d150.18a900ea986fbf2b namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 48da9fa6-64e4-493a-9ff6-83c9b6cc90d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8d150 namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: ad1e5b45-f096-44c0-bda8-da5192b401e1 kind: Event lastTimestamp: "2026-04-23T13:55:35Z" message: InferenceGraph [sequence-graph-8d150] is Ready metadata: creationTimestamp: "2026-04-23T13:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:55:35Z" name: sequence-graph-8d150.18a900ea9be27516 namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: 0fb122f8-137f-431e-9b2a-e70a6c2d91cf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:55:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d150 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: 1d64f8e5-3c44-47dd-853b-42c0fdaff046 kind: Event lastTimestamp: "2026-04-23T13:56:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:56:05Z" name: sequence-graph-8d150.18a900ee1709001b namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 2c924f90-1ca7-4512-8015-11144a60a1c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:55:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d150 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: 1d64f8e5-3c44-47dd-853b-42c0fdaff046 kind: Event lastTimestamp: "2026-04-23T13:56:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:56:05Z" name: sequence-graph-8d150.18a900ee1769b8c0 namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: aab4170f-7076-4cfb-8b0b-e14d577e43ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:45:10.388897Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f7bdd-7ccdd96579-trdwk namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: ae2dabb8-a691-4df0-bcf8-b03f712dbf61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f7bdd-7ccdd96579-trdwk to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:45:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:45:10Z" name: sequence-graph-f7bdd-7ccdd96579-trdwk.18a9005909aeb466 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 1b995f2f-4177-4e28-94f7-41476b819344 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:10Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f7bdd-7ccdd96579-trdwk namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: ae2dabb8-a691-4df0-bcf8-b03f712dbf61 kind: Event lastTimestamp: "2026-04-23T13:45:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-f7bdd-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:10Z" name: sequence-graph-f7bdd-7ccdd96579-trdwk.18a90059164fcb05 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: 9909009d-a014-47e5-a8d6-ab1eda4308ea reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f7bdd-7ccdd96579-trdwk namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: ae2dabb8-a691-4df0-bcf8-b03f712dbf61 kind: Event lastTimestamp: "2026-04-23T13:45:11Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:45:11Z" name: sequence-graph-f7bdd-7ccdd96579-trdwk.18a900594647fd73 namespace: kserve-ci-e2e-test resourceVersion: "16114" uid: d8831912-c240-4e76-a16a-687da62c27a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7bdd} kind: Pod name: sequence-graph-f7bdd-7ccdd96579-trdwk namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: ae2dabb8-a691-4df0-bcf8-b03f712dbf61 kind: Event lastTimestamp: "2026-04-23T13:45:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T13:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:11Z" name: sequence-graph-f7bdd-7ccdd96579-trdwk.18a90059479ac289 namespace: kserve-ci-e2e-test resourceVersion: "16116" uid: c87661f6-5012-4cc4-9b00-300e1f7f8b43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7bdd} kind: Pod name: sequence-graph-f7bdd-7ccdd96579-trdwk namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: ae2dabb8-a691-4df0-bcf8-b03f712dbf61 kind: Event lastTimestamp: "2026-04-23T13:45:11Z" message: 'Created container: sequence-graph-f7bdd' metadata: creationTimestamp: "2026-04-23T13:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:11Z" name: sequence-graph-f7bdd-7ccdd96579-trdwk.18a900594a96777d namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: be1ee8df-0981-4067-8b70-894f4fdd1693 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7bdd} kind: Pod name: sequence-graph-f7bdd-7ccdd96579-trdwk namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: ae2dabb8-a691-4df0-bcf8-b03f712dbf61 kind: Event lastTimestamp: "2026-04-23T13:45:11Z" message: Started container sequence-graph-f7bdd metadata: creationTimestamp: "2026-04-23T13:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:11Z" name: sequence-graph-f7bdd-7ccdd96579-trdwk.18a900594b07d103 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: 728453c6-c0ce-454f-900b-05f145927dc7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7bdd} kind: Pod name: sequence-graph-f7bdd-7ccdd96579-trdwk namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: ae2dabb8-a691-4df0-bcf8-b03f712dbf61 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Stopping container sequence-graph-f7bdd metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: sequence-graph-f7bdd-7ccdd96579-trdwk.18a900cc390191d9 namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: 09e6a687-644c-434e-83c6-b957ace24aef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:53:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7bdd} kind: Pod name: sequence-graph-f7bdd-7ccdd96579-trdwk namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: ae2dabb8-a691-4df0-bcf8-b03f712dbf61 kind: Event lastTimestamp: "2026-04-23T13:53:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:53:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:52Z" name: sequence-graph-f7bdd-7ccdd96579-trdwk.18a900ccd72787db namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: b4eb9b8e-6929-47cc-b9b9-d47d0c396bbc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f7bdd-7ccdd96579 namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: ff75c275-3a2f-440e-9f32-956a9136cc3b kind: Event lastTimestamp: "2026-04-23T13:45:10Z" message: 'Created pod: sequence-graph-f7bdd-7ccdd96579-trdwk' metadata: creationTimestamp: "2026-04-23T13:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:10Z" name: sequence-graph-f7bdd-7ccdd96579.18a900590948afee namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 63742f8e-ffc0-42ac-8989-aee8d105d1e3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 02b73213-6216-44a1-8af1-fb982e8bd69f kind: Event lastTimestamp: "2026-04-23T13:45:10Z" message: Scaled up replica set sequence-graph-f7bdd-7ccdd96579 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:10Z" name: sequence-graph-f7bdd.18a900590868c7d3 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 15e0220e-d8b3-4ba2-ab78-35898038a83d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: 50c51f2c-a05d-4e4c-9e23-bf01f81b0f18 kind: Event lastTimestamp: "2026-04-23T13:45:10Z" message: InferenceGraph [sequence-graph-f7bdd] is Ready metadata: creationTimestamp: "2026-04-23T13:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:10Z" name: sequence-graph-f7bdd.18a900590d3e10e7 namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: aa152ebd-1d67-44f6-8fdf-842745ce2f43 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 715a55a2-5b3d-4bb6-b32c-146b4d4a6249 kind: Event lastTimestamp: "2026-04-23T13:45:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:40Z" name: sequence-graph-f7bdd.18a9005c86bce1e8 namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: fb7959e9-a4b8-4664-9222-327e2dbab044 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 715a55a2-5b3d-4bb6-b32c-146b4d4a6249 kind: Event lastTimestamp: "2026-04-23T13:45:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:40Z" name: sequence-graph-f7bdd.18a9005c86fd5bcc namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 22d64b35-621f-40f4-a0c6-ecc4989fd962 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T14:04:14.771849Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5035e-5c58fb744-qcjf5 namespace: kserve-ci-e2e-test resourceVersion: "25326" uid: 9cc3527f-b5d1-4e0c-bb4c-08b69b815709 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-5035e-5c58fb744-qcjf5 to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T14:04:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T14:04:14Z" name: splitter-graph-5035e-5c58fb744-qcjf5.18a901637c38cea1 namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: 40fbb581-8845-46e2-b24b-fb7b2e8c4168 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:15Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5035e-5c58fb744-qcjf5 namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: 9cc3527f-b5d1-4e0c-bb4c-08b69b815709 kind: Event lastTimestamp: "2026-04-23T14:04:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-5035e-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T14:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:15Z" name: splitter-graph-5035e-5c58fb744-qcjf5.18a901638a6143cd namespace: kserve-ci-e2e-test resourceVersion: "25345" uid: 696a0ccf-18d3-4670-99f1-287d371527df reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:15Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5035e-5c58fb744-qcjf5 namespace: kserve-ci-e2e-test resourceVersion: "25336" uid: 9cc3527f-b5d1-4e0c-bb4c-08b69b815709 kind: Event lastTimestamp: "2026-04-23T14:04:15Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T14:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T14:04:15Z" name: splitter-graph-5035e-5c58fb744-qcjf5.18a90163b8cbc1a3 namespace: kserve-ci-e2e-test resourceVersion: "25355" uid: 341ea163-8350-48ef-b908-0d46c816cd8c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5035e} kind: Pod name: splitter-graph-5035e-5c58fb744-qcjf5 namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: 9cc3527f-b5d1-4e0c-bb4c-08b69b815709 kind: Event lastTimestamp: "2026-04-23T14:04:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T14:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:15Z" name: splitter-graph-5035e-5c58fb744-qcjf5.18a90163ba08ce47 namespace: kserve-ci-e2e-test resourceVersion: "25357" uid: 25734802-94ba-4b03-ba71-c299c10b3023 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5035e} kind: Pod name: splitter-graph-5035e-5c58fb744-qcjf5 namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: 9cc3527f-b5d1-4e0c-bb4c-08b69b815709 kind: Event lastTimestamp: "2026-04-23T14:04:15Z" message: 'Created container: splitter-graph-5035e' metadata: creationTimestamp: "2026-04-23T14:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:15Z" name: splitter-graph-5035e-5c58fb744-qcjf5.18a90163bd6005e8 namespace: kserve-ci-e2e-test resourceVersion: "25358" uid: fa95853b-4c80-4039-9c2c-161d73187c21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5035e} kind: Pod name: splitter-graph-5035e-5c58fb744-qcjf5 namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: 9cc3527f-b5d1-4e0c-bb4c-08b69b815709 kind: Event lastTimestamp: "2026-04-23T14:04:15Z" message: Started container splitter-graph-5035e metadata: creationTimestamp: "2026-04-23T14:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:15Z" name: splitter-graph-5035e-5c58fb744-qcjf5.18a90163bdd11fac namespace: kserve-ci-e2e-test resourceVersion: "25359" uid: d647a231-dcb2-4649-a5e0-c7443a2046d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5035e} kind: Pod name: splitter-graph-5035e-5c58fb744-qcjf5 namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: 9cc3527f-b5d1-4e0c-bb4c-08b69b815709 kind: Event lastTimestamp: "2026-04-23T14:04:24Z" message: Stopping container splitter-graph-5035e metadata: creationTimestamp: "2026-04-23T14:04:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:24Z" name: splitter-graph-5035e-5c58fb744-qcjf5.18a90165d53a959d namespace: kserve-ci-e2e-test resourceVersion: "25457" uid: 2fb3114b-7687-4243-82cd-8c5b0a78227f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:04:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5035e} kind: Pod name: splitter-graph-5035e-5c58fb744-qcjf5 namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: 9cc3527f-b5d1-4e0c-bb4c-08b69b815709 kind: Event lastTimestamp: "2026-04-23T14:04:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T14:04:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:52Z" name: splitter-graph-5035e-5c58fb744-qcjf5.18a901665648bc3c namespace: kserve-ci-e2e-test resourceVersion: "25856" uid: 64a63532-7d1d-44e6-8d68-bcf7ce513e99 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-5035e-5c58fb744 namespace: kserve-ci-e2e-test resourceVersion: "25322" uid: a13d70f2-2328-441f-97a5-1a1cdaa35577 kind: Event lastTimestamp: "2026-04-23T14:04:14Z" message: 'Created pod: splitter-graph-5035e-5c58fb744-qcjf5' metadata: creationTimestamp: "2026-04-23T14:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:14Z" name: splitter-graph-5035e-5c58fb744.18a901637baa9441 namespace: kserve-ci-e2e-test resourceVersion: "25328" uid: 48374cea-9dbc-4bc3-b562-c6f3af77cd1b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-5035e namespace: kserve-ci-e2e-test resourceVersion: "25320" uid: 8b4d3c90-c620-49dd-9aba-688b911922cc kind: Event lastTimestamp: "2026-04-23T14:04:14Z" message: Scaled up replica set splitter-graph-5035e-5c58fb744 from 0 to 1 metadata: creationTimestamp: "2026-04-23T14:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:14Z" name: splitter-graph-5035e.18a901637af97906 namespace: kserve-ci-e2e-test resourceVersion: "25324" uid: 8c0519f7-6fc0-4d39-80d9-9721516676e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-5035e namespace: kserve-ci-e2e-test resourceVersion: "25340" uid: a6743100-7e2f-4c9b-90fc-152168321123 kind: Event lastTimestamp: "2026-04-23T14:04:14Z" message: InferenceGraph [splitter-graph-5035e] is Ready metadata: creationTimestamp: "2026-04-23T14:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:14Z" name: splitter-graph-5035e.18a9016380030e58 namespace: kserve-ci-e2e-test resourceVersion: "25342" uid: 35fca080-b874-4dc7-9353-35d0e5e4225a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T14:05:25.069867Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a8c3e-bc877ccbf-qwvt7 namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: fca53ba5-e5a9-4af1-8910-72a0f613d62e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a8c3e-bc877ccbf-qwvt7 to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T14:05:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T14:05:25Z" name: splitter-graph-a8c3e-bc877ccbf-qwvt7.18a90173da4f6de7 namespace: kserve-ci-e2e-test resourceVersion: "26449" uid: ac54a3a2-f97d-4181-9c9b-ebb6313a4769 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:25Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a8c3e-bc877ccbf-qwvt7 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: fca53ba5-e5a9-4af1-8910-72a0f613d62e kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-a8c3e-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T14:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:05:25Z" name: splitter-graph-a8c3e-bc877ccbf-qwvt7.18a90173e44c3c0a namespace: kserve-ci-e2e-test resourceVersion: "26462" uid: 3d762fb7-051a-4cf7-81d0-046a55ca8e2b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:26Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a8c3e-bc877ccbf-qwvt7 namespace: kserve-ci-e2e-test resourceVersion: "26452" uid: fca53ba5-e5a9-4af1-8910-72a0f613d62e kind: Event lastTimestamp: "2026-04-23T14:05:26Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T14:05:26Z" name: splitter-graph-a8c3e-bc877ccbf-qwvt7.18a90174171ba0ab namespace: kserve-ci-e2e-test resourceVersion: "26471" uid: d5923592-be4b-4d97-86d1-0774bab3fa0d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a8c3e} kind: Pod name: splitter-graph-a8c3e-bc877ccbf-qwvt7 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: fca53ba5-e5a9-4af1-8910-72a0f613d62e kind: Event lastTimestamp: "2026-04-23T14:05:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:05:26Z" name: splitter-graph-a8c3e-bc877ccbf-qwvt7.18a9017418a176e3 namespace: kserve-ci-e2e-test resourceVersion: "26473" uid: 3520fd80-e995-42ed-9555-73c71ac1f468 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a8c3e} kind: Pod name: splitter-graph-a8c3e-bc877ccbf-qwvt7 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: fca53ba5-e5a9-4af1-8910-72a0f613d62e kind: Event lastTimestamp: "2026-04-23T14:05:26Z" message: 'Created container: splitter-graph-a8c3e' metadata: creationTimestamp: "2026-04-23T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:05:26Z" name: splitter-graph-a8c3e-bc877ccbf-qwvt7.18a901741b94ae4f namespace: kserve-ci-e2e-test resourceVersion: "26474" uid: c5cd7de3-9f3d-4774-bf80-3dbc15f6c3eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a8c3e} kind: Pod name: splitter-graph-a8c3e-bc877ccbf-qwvt7 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: fca53ba5-e5a9-4af1-8910-72a0f613d62e kind: Event lastTimestamp: "2026-04-23T14:05:26Z" message: Started container splitter-graph-a8c3e metadata: creationTimestamp: "2026-04-23T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:05:26Z" name: splitter-graph-a8c3e-bc877ccbf-qwvt7.18a901741c027d99 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 7198116d-a741-4ba1-bcd7-b134fd88e208 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a8c3e} kind: Pod name: splitter-graph-a8c3e-bc877ccbf-qwvt7 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: fca53ba5-e5a9-4af1-8910-72a0f613d62e kind: Event lastTimestamp: "2026-04-23T14:13:39Z" message: Stopping container splitter-graph-a8c3e metadata: creationTimestamp: "2026-04-23T14:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:13:39Z" name: splitter-graph-a8c3e-bc877ccbf-qwvt7.18a901e705b4200f namespace: kserve-ci-e2e-test resourceVersion: "29910" uid: 4c351a55-b001-45c2-a0b3-0bf3ec1371b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:13:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a8c3e} kind: Pod name: splitter-graph-a8c3e-bc877ccbf-qwvt7 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: fca53ba5-e5a9-4af1-8910-72a0f613d62e kind: Event lastTimestamp: "2026-04-23T14:14:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T14:13:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:14:07Z" name: splitter-graph-a8c3e-bc877ccbf-qwvt7.18a901e79c492bb7 namespace: kserve-ci-e2e-test resourceVersion: "30141" uid: 7637aff7-c617-4dca-bcbc-c271386e1e5e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a8c3e-bc877ccbf namespace: kserve-ci-e2e-test resourceVersion: "26440" uid: 52376852-319f-46eb-af65-38d64523c585 kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: 'Created pod: splitter-graph-a8c3e-bc877ccbf-qwvt7' metadata: creationTimestamp: "2026-04-23T14:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:25Z" name: splitter-graph-a8c3e-bc877ccbf.18a90173d9f8373e namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: 486d653e-3201-47bf-8891-b5ecbe743fda reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: d4a89656-a74c-49aa-8c8e-b68685248e40 kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: Scaled up replica set splitter-graph-a8c3e-bc877ccbf from 0 to 1 metadata: creationTimestamp: "2026-04-23T14:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:25Z" name: splitter-graph-a8c3e.18a90173d92b6944 namespace: kserve-ci-e2e-test resourceVersion: "26441" uid: b5ea83a6-0582-4381-9da9-c8408ffee972 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: 266465b1-8091-4f0e-bd66-3edda946a7ed kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: InferenceGraph [splitter-graph-a8c3e] is Ready metadata: creationTimestamp: "2026-04-23T14:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:05:25Z" name: splitter-graph-a8c3e.18a90173df6b37ac namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: d9196ee6-4016-4c46-a8ef-3195f08dc03a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: 266465b1-8091-4f0e-bd66-3edda946a7ed kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: 'Failed to update status for InferenceGraph "splitter-graph-a8c3e": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-a8c3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:05:25Z" name: splitter-graph-a8c3e.18a90173e24965c1 namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: 3f0feab2-99c0-4995-896f-c42f92625797 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: 266465b1-8091-4f0e-bd66-3edda946a7ed kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-a8c3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:05:25Z" name: splitter-graph-a8c3e.18a90173e249b4b8 namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 154b5682-68e2-4944-a831-e67a9257becc reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:05:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "26439" uid: cb9fa16f-acfd-4962-b215-6a48385e21cb kind: Event lastTimestamp: "2026-04-23T14:05:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:55Z" name: splitter-graph-a8c3e.18a9017757645a14 namespace: kserve-ci-e2e-test resourceVersion: "26696" uid: e4296ddb-a2df-4e5f-bff2-170d56fd6ab4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:05:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "26439" uid: cb9fa16f-acfd-4962-b215-6a48385e21cb kind: Event lastTimestamp: "2026-04-23T14:05:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:55Z" name: splitter-graph-a8c3e.18a9017757c6d5ee namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: f79b9dd5-d1c7-457c-807b-8058a62b6d59 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:53:25.266288Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:53:25Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cc42ac6302 namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: 510cfc60-0eff-44c4-819b-0c9b13335269 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1cfcc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cc52eb261e namespace: kserve-ci-e2e-test resourceVersion: "19791" uid: bfaf9269-7806-4c8b-be7a-3a3eec072a10 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:53:26Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:53:26Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cc8007db4a namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 3320cba7-d985-4c0f-b83e-ca8bbde119ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:53:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:26Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cc81558659 namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 06b8402c-92bb-408d-993d-a2885a4ad21d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:53:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:26Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cc84a8537c namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 03b4185c-6bb5-4090-af74-390bc98c7720 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:53:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:26Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cc851c11e2 namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: c459fddd-a0ac-43c8-b46f-f849c8199244 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:53:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:26Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cc85298011 namespace: kserve-ci-e2e-test resourceVersion: "19817" uid: 1e72e62d-6b28-4302-b1a5-85896f5a094c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:53:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:26Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cc8893a089 namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: 83862b99-6e02-47de-b11e-4e2025b7c896 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:53:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:26Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cc8907bf2b namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 65b01023-af73-44dd-b386-dcf926f12518 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:53:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:54:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:53:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:04Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900cce5ca5256 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: ffd4d0e1-908f-4993-ab1d-2a00fd7e7222 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:35Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900dc9c5526dc namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: 39fc45b8-49b0-47d9-aa89-668ad0a45aed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f4f2fe6e-2e3d-422b-9e9c-8a4fbd26be00 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:35Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv.18a900dc9c55b03f namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 76e15856-7289-4745-a1eb-aae554f6034a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1cfcc-predictor-5b5dd86bb9 namespace: kserve-ci-e2e-test resourceVersion: "19726" uid: a34dfc45-be84-49ef-aa70-b4c6af4287d8 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: 'Created pod: success-200-isvc-1cfcc-predictor-5b5dd86bb9-h48hv' metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:25Z" name: success-200-isvc-1cfcc-predictor-5b5dd86bb9.18a900cc42211a31 namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 76a68c70-0c04-4664-a3e1-d0963943955f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: c569dbb0-4b93-4b23-8621-89b1fabf4691 kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Scaled up replica set success-200-isvc-1cfcc-predictor-5b5dd86bb9 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:25Z" name: success-200-isvc-1cfcc-predictor.18a900cc40112926 namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: 76dc4b78-4529-4389-9c5b-4fc397651791 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:53:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: a8f005c6-c850-453b-8b1e-2a0ae44ce4dd kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:25Z" name: success-200-isvc-1cfcc-predictor.18a900cfbee83f53 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 3820a3aa-09d0-4c85-9af5-fc45992f78a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:53:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: a8f005c6-c850-453b-8b1e-2a0ae44ce4dd kind: Event lastTimestamp: "2026-04-23T13:54:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:25Z" name: success-200-isvc-1cfcc-predictor.18a900cfbf70cbaa namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 2200750d-600a-4a72-9626-f23500421d2d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:53:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: a8f005c6-c850-453b-8b1e-2a0ae44ce4dd kind: Event lastTimestamp: "2026-04-23T13:54:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:10Z" name: success-200-isvc-1cfcc-predictor.18a900d33e52641e namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: ea54666c-1e49-4de3-ba32-1a1537160322 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:53:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1cfcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: a8f005c6-c850-453b-8b1e-2a0ae44ce4dd kind: Event lastTimestamp: "2026-04-23T13:54:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:10Z" name: success-200-isvc-1cfcc-predictor.18a900d33edb9615 namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 91a3da1e-fa8e-436c-bf47-169ece5d6b6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: 420c4d04-1564-430a-8d29-8dee7d26c26e kind: Event lastTimestamp: "2026-04-23T13:54:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:14Z" name: success-200-isvc-1cfcc.18a900cc3fd6a730 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 58ca1187-ea3c-4f47-8f15-2c7d800356bb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: 420c4d04-1564-430a-8d29-8dee7d26c26e kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: 'Failed to update status for InferenceService "success-200-isvc-1cfcc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1cfcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:25Z" name: success-200-isvc-1cfcc.18a900cc404e4eb3 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: aeff3fb8-7537-48fd-9be7-a5f08a6a6ae0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: 420c4d04-1564-430a-8d29-8dee7d26c26e kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1cfcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:25Z" name: success-200-isvc-1cfcc.18a900cc404e9d32 namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: 8e779984-3de2-4453-b2a6-9f75b1a4f96e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1cfcc namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 420c4d04-1564-430a-8d29-8dee7d26c26e kind: Event lastTimestamp: "2026-04-23T13:54:14Z" message: InferenceService [success-200-isvc-1cfcc] is Ready metadata: creationTimestamp: "2026-04-23T13:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:14Z" name: success-200-isvc-1cfcc.18a900d7a1c381a4 namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: aab71318-ebea-4c5f-967e-481f009f1c3d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:52:49.491545Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:52:49Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c3ee553844 namespace: kserve-ci-e2e-test resourceVersion: "19340" uid: 261ccd1e-4600-4507-afbb-0283f9fd7d7d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2d7b2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:49Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c3f6a7fc4c namespace: kserve-ci-e2e-test resourceVersion: "19370" uid: dbdcc83b-f385-4787-be07-195cb5fbc7fb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19349" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:52:50Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c42b3f6a3a namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: d7cd3243-7902-4b66-93b6-14bf391b1b2f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c42c8e946c namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: 4b2555cb-7efc-43b7-b475-88d82784e9c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c42f84e4f4 namespace: kserve-ci-e2e-test resourceVersion: "19399" uid: 82e67803-efb7-4966-816b-2a816fdca7a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c42ffa916d namespace: kserve-ci-e2e-test resourceVersion: "19400" uid: 4e1e9764-f9ed-48ec-9f72-4ce09af61d44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c43006c067 namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: 019122ac-dbca-4076-96ed-de1877ba10e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c432fb6787 namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: 9efbc4de-f3ba-48bb-8980-62e0bb25673c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:52:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:50Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c433700ab8 namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: b1e4da3c-b698-4ff8-a6fb-de5bfad01479 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:52:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:53:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:27Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900c47d4a0f1b namespace: kserve-ci-e2e-test resourceVersion: "19835" uid: 9eefdab6-7cee-47e7-8284-662bc96535b6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:59Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900d44c0dfb32 namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: 0bb4b65b-ac4a-4f68-b5d4-4cbd851baa09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:59Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900d44c0e9c0a namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: 5d030f1d-d687-45ad-8fcc-753cc0b06052 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 392cb698-8a42-421c-bc37-5884e4f47df1 kind: Event lastTimestamp: "2026-04-23T13:54:02Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:02Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g.18a900d50575a0c7 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 5696e1e9-6606-4d2a-8fb7-21de4a0c45fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2d7b2-predictor-5885f7c96c namespace: kserve-ci-e2e-test resourceVersion: "19310" uid: b4ba9375-6b63-44ca-afb7-e2b0222d5184 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: 'Created pod: success-200-isvc-2d7b2-predictor-5885f7c96c-tld6g' metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:52:49Z" name: success-200-isvc-2d7b2-predictor-5885f7c96c.18a900c3ede90f63 namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: 56ece62a-062e-4c9f-9038-fccfb4079b87 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: 6e786c1f-1263-4968-9b82-0b689cb37820 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: Scaled up replica set success-200-isvc-2d7b2-predictor-5885f7c96c from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:52:49Z" name: success-200-isvc-2d7b2-predictor.18a900c3ea7615b5 namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: 9e94bfca-ac78-4e03-a150-8ffb2c76eae5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:53:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 972a0417-0b43-4fc5-bef9-a013f7015d5d kind: Event lastTimestamp: "2026-04-23T13:53:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:34Z" name: success-200-isvc-2d7b2-predictor.18a900c768ed0164 namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: 17a785f4-9f6a-413c-8619-d14ffed4f4fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:53:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 972a0417-0b43-4fc5-bef9-a013f7015d5d kind: Event lastTimestamp: "2026-04-23T13:53:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:34Z" name: success-200-isvc-2d7b2-predictor.18a900c76951ab7e namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: 949367f2-fba6-49f3-9351-56de045f00dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19523" uid: 972a0417-0b43-4fc5-bef9-a013f7015d5d kind: Event lastTimestamp: "2026-04-23T13:53:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:49Z" name: success-200-isvc-2d7b2-predictor.18a900d1e5da9f66 namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: de277846-bba2-42ec-8e6b-dc9f8db44a3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d7b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19523" uid: 972a0417-0b43-4fc5-bef9-a013f7015d5d kind: Event lastTimestamp: "2026-04-23T13:53:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:49Z" name: success-200-isvc-2d7b2-predictor.18a900d1e6a7e38a namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: e0ac3024-40f7-4e16-acc6-45b22faf6f24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: f502f3be-4314-49f2-b94a-8cd8a1d13c1a kind: Event lastTimestamp: "2026-04-23T13:53:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:38Z" name: success-200-isvc-2d7b2.18a900c3ea455d0d namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: fa1ddb62-afe4-415a-9fda-93c8b4dffc56 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: f502f3be-4314-49f2-b94a-8cd8a1d13c1a kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: 'Failed to update status for InferenceService "success-200-isvc-2d7b2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2d7b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:52:49Z" name: success-200-isvc-2d7b2.18a900c3eaa042cf namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 7e58b757-74eb-4797-bf3d-74494a9e7aa0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: f502f3be-4314-49f2-b94a-8cd8a1d13c1a kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2d7b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:52:49Z" name: success-200-isvc-2d7b2.18a900c3eaa09bf9 namespace: kserve-ci-e2e-test resourceVersion: "19316" uid: 85ccdc70-5164-4b64-ac93-e542b357584c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2d7b2 namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: f502f3be-4314-49f2-b94a-8cd8a1d13c1a kind: Event lastTimestamp: "2026-04-23T13:53:37Z" message: InferenceService [success-200-isvc-2d7b2] is Ready metadata: creationTimestamp: "2026-04-23T13:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:37Z" name: success-200-isvc-2d7b2.18a900cf385930ee namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 56a9d6de-b595-4413-b70d-1b20f7a914b5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:53:59.829756Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-37fbd-predictor-8f56b4685-mbxhd to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:53:59Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a900d44ed10e1f namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: d19e990a-38c4-4eda-8470-a817cee6d908 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:54:00Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a900d467f5d6bb namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: a96e8802-840f-4276-9925-f570bc7552ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a900d469779ce5 namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: 666b168f-f35f-4797-94c3-9f20f742c3d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a900d46ce10884 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: 029edf99-491c-493c-bb98-71c3b4e2b7a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a900d46d5b63dc namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 80fcb5c3-5b99-4653-9b74-e5de8db696ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a900d46d672994 namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 55ec9a69-0cb1-45e5-b355-c6bc7832a1cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a900d46ff5eca8 namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: a39ce327-4a69-4b0d-aad9-62776dcf60bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T13:54:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:00Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a900d47091b361 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: 765fa8ee-ccd3-40e3-a443-251076684d48 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T13:54:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:38Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a900d4d6b51393 namespace: kserve-ci-e2e-test resourceVersion: "20807" uid: 492e65cb-857e-4d42-b06f-0eeb5c43f8e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:14Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a901557ba73bde namespace: kserve-ci-e2e-test resourceVersion: "24585" uid: 5b06193e-e000-4f4f-8999-e32a1c209ff0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 3d08b04c-3401-4df1-b1c5-82a8543e716d kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:14Z" name: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd.18a901557ba88058 namespace: kserve-ci-e2e-test resourceVersion: "24588" uid: 662c2bcf-f724-4de0-ab5d-ce05a27baddc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-37fbd-predictor-8f56b4685 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: f5b08841-bf45-40ed-92f4-f4971e3003e3 kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: 'Created pod: success-200-isvc-37fbd-predictor-8f56b4685-mbxhd' metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:59Z" name: success-200-isvc-37fbd-predictor-8f56b4685.18a900d44e01024a namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: 31f8c1cd-55af-4bb7-8963-2abb44c3a492 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 865526e5-d506-4233-b3f2-73ac33282cad kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: Scaled up replica set success-200-isvc-37fbd-predictor-8f56b4685 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:59Z" name: success-200-isvc-37fbd-predictor.18a900d44a81f725 namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 6a9c5f10-1913-4e0f-a600-13f256d12fc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:54:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: b8cb30fa-98b2-43c3-b3c1-337c947da933 kind: Event lastTimestamp: "2026-04-23T13:54:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:29Z" name: success-200-isvc-37fbd-predictor.18a900d7c924ee28 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 57c325d4-ae4f-4083-8578-06b2a4903daa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:54:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: b8cb30fa-98b2-43c3-b3c1-337c947da933 kind: Event lastTimestamp: "2026-04-23T13:54:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:29Z" name: success-200-isvc-37fbd-predictor.18a900d7c98bb728 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 7af0334e-e4a8-4e14-a237-b4985bc70a9b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: b8cb30fa-98b2-43c3-b3c1-337c947da933 kind: Event lastTimestamp: "2026-04-23T13:55:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:14Z" name: success-200-isvc-37fbd-predictor.18a900dec75ca2bf namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: 67c30c16-d64b-4d55-bdb9-da7f594d361d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-37fbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: b8cb30fa-98b2-43c3-b3c1-337c947da933 kind: Event lastTimestamp: "2026-04-23T13:55:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:14Z" name: success-200-isvc-37fbd-predictor.18a900dec7b58421 namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: c1dd5221-ed36-4c10-be9f-9580f75e08e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-37fbd namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: f17424d2-db55-4dab-8894-d811d156844e kind: Event lastTimestamp: "2026-04-23T13:54:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:48Z" name: success-200-isvc-37fbd.18a900d44a162f24 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: df1e99f6-f74c-4b2b-94f2-a6f735c539b1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-37fbd namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: f17424d2-db55-4dab-8894-d811d156844e kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: 'Failed to update status for InferenceService "success-200-isvc-37fbd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-37fbd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:59Z" name: success-200-isvc-37fbd.18a900d44ac5517b namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: 91e5b3ae-10d8-4840-bcd4-0db5954255aa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-37fbd namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: f17424d2-db55-4dab-8894-d811d156844e kind: Event lastTimestamp: "2026-04-23T13:53:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-37fbd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:53:59Z" name: success-200-isvc-37fbd.18a900d44ac5bbe0 namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 729297e1-945a-4fd4-bb45-4695329fd455 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-37fbd namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: f17424d2-db55-4dab-8894-d811d156844e kind: Event lastTimestamp: "2026-04-23T13:54:48Z" message: InferenceService [success-200-isvc-37fbd] is Ready metadata: creationTimestamp: "2026-04-23T13:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:48Z" name: success-200-isvc-37fbd.18a900df93c451e6 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: b8b14aaa-7205-4d9a-9279-75fd81c4e664 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T14:03:14.729513Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24614" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T14:03:14Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a90155816b82e1 namespace: kserve-ci-e2e-test resourceVersion: "24622" uid: cbe8ae09-26b2-4cb6-97e2-1cab40d0d1b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24633" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T14:03:15Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a901559acd0cc0 namespace: kserve-ci-e2e-test resourceVersion: "24659" uid: 50d368a2-29bb-4eee-8928-f78e5377083f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a901559c3d4acc namespace: kserve-ci-e2e-test resourceVersion: "24661" uid: 7650e91f-ed1d-4f1b-ac0e-9a15b62c9751 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a901559fc9bc8a namespace: kserve-ci-e2e-test resourceVersion: "24664" uid: 571ec20e-ae69-44b9-8721-bfc82fb68a65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a90155a06afc81 namespace: kserve-ci-e2e-test resourceVersion: "24666" uid: 9bb1ac26-a465-45bb-8812-1168f8c06370 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a90155a078c710 namespace: kserve-ci-e2e-test resourceVersion: "24668" uid: ab2f740c-25e2-4359-9b0d-5512851661b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a90155a360290d namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: 94668b48-365b-45a8-870d-6d3b1798b8a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:03:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:15Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a90155a3d4805e namespace: kserve-ci-e2e-test resourceVersion: "24671" uid: c02b6d06-1117-4cce-b131-9c5b1d4d6b78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:03:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:03:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T14:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:52Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a90155f870db85 namespace: kserve-ci-e2e-test resourceVersion: "25108" uid: e16cc2d3-41fc-4759-8dfe-aa86452ce455 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:04:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:04:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:24Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a90165dcb17f09 namespace: kserve-ci-e2e-test resourceVersion: "25504" uid: 88c195cc-36c5-4afa-8582-2113b051bde9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:04:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:04:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:24Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a90165dcb225ac namespace: kserve-ci-e2e-test resourceVersion: "25505" uid: 6c6523e4-9462-4567-8e56-d25d1275bfc1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 4ed1d85a-d2dc-4b9d-b42f-de90afc04a20 kind: Event lastTimestamp: "2026-04-23T14:04:27Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T14:04:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:27Z" name: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s.18a901668091883b namespace: kserve-ci-e2e-test resourceVersion: "25595" uid: 556249ee-4f9a-4a0e-a886-536580a98135 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5035e-predictor-6cc7687ff7 namespace: kserve-ci-e2e-test resourceVersion: "24590" uid: 6ee7a5b9-0444-4f14-918c-e8ada5f62930 kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: 'Created pod: success-200-isvc-5035e-predictor-6cc7687ff7-sbk7s' metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:14Z" name: success-200-isvc-5035e-predictor-6cc7687ff7.18a9015580da2aff namespace: kserve-ci-e2e-test resourceVersion: "24617" uid: a19a7830-3cfa-4c6a-9755-1e6050fe8ed5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: 1b712a64-400b-4481-bd80-8f20694debaf kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: Scaled up replica set success-200-isvc-5035e-predictor-6cc7687ff7 from 0 to 1 metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:14Z" name: success-200-isvc-5035e-predictor.18a901557cb664ad namespace: kserve-ci-e2e-test resourceVersion: "24592" uid: ca987801-f4cd-4ffa-9802-039c069771d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24589" uid: c7784a68-74fa-4fa5-8ec9-4644b63ed23d kind: Event lastTimestamp: "2026-04-23T14:03:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:44Z" name: success-200-isvc-5035e-predictor.18a90158fb52b7d2 namespace: kserve-ci-e2e-test resourceVersion: "24913" uid: 7ca45e25-d77e-4994-bb27-cf6bc8017bbe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24589" uid: c7784a68-74fa-4fa5-8ec9-4644b63ed23d kind: Event lastTimestamp: "2026-04-23T14:03:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:44Z" name: success-200-isvc-5035e-predictor.18a90158fbbc00dd namespace: kserve-ci-e2e-test resourceVersion: "24914" uid: cb0421a0-08fe-473f-85fe-749116a409ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24800" uid: c7784a68-74fa-4fa5-8ec9-4644b63ed23d kind: Event lastTimestamp: "2026-04-23T14:04:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:14Z" name: success-200-isvc-5035e-predictor.18a9015ff938506e namespace: kserve-ci-e2e-test resourceVersion: "25312" uid: 459785ae-e104-4498-8a27-fb9c060a5522 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5035e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24800" uid: c7784a68-74fa-4fa5-8ec9-4644b63ed23d kind: Event lastTimestamp: "2026-04-23T14:04:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:14Z" name: success-200-isvc-5035e-predictor.18a9015ff99fe9bf namespace: kserve-ci-e2e-test resourceVersion: "25314" uid: 7e8315ba-2065-4eea-8ccd-b09630ff74a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5035e namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 6393abda-9db7-4cd0-adba-5d55362ead71 kind: Event lastTimestamp: "2026-04-23T14:04:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:02Z" name: success-200-isvc-5035e.18a901557c96f0bc namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: b838c6b5-72ec-45a9-a979-f97a7fa0e7b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5035e namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 6393abda-9db7-4cd0-adba-5d55362ead71 kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: 'Failed to update status for InferenceService "success-200-isvc-5035e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5035e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:03:14Z" name: success-200-isvc-5035e.18a901557d0d3fa8 namespace: kserve-ci-e2e-test resourceVersion: "24594" uid: d4a8b9c5-a27e-4038-9521-dca5494462d0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5035e namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 6393abda-9db7-4cd0-adba-5d55362ead71 kind: Event lastTimestamp: "2026-04-23T14:03:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5035e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:03:14Z" name: success-200-isvc-5035e.18a901557d0d896d namespace: kserve-ci-e2e-test resourceVersion: "24596" uid: cc61a5ad-0976-490c-bd0b-6f1fa066b760 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5035e namespace: kserve-ci-e2e-test resourceVersion: "25209" uid: 6393abda-9db7-4cd0-adba-5d55362ead71 kind: Event lastTimestamp: "2026-04-23T14:04:02Z" message: InferenceService [success-200-isvc-5035e] is Ready metadata: creationTimestamp: "2026-04-23T14:04:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:02Z" name: success-200-isvc-5035e.18a90160b3939f92 namespace: kserve-ci-e2e-test resourceVersion: "25210" uid: ba20b9c7-6f29-4737-835a-8a62bdd9a2d3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:54:35.509892Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8d150-predictor-5cd95d7749-hpnmb to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:54:35Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dc9d84ccc3 namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: 7f0f20dc-bcfd-40ef-9bc8-8c78fff1f331 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8d150-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:35Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dcaa6d43ac namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 79ada672-b158-4491-b676-45b53d7c4c2b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:54:36Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dcdbfe780b namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: 1ad799cc-2276-44d5-8d6b-71b3720dc8c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dcdd4040ed namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 7352b1ba-11b0-40cd-ab75-f327d9227b2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dce0398a40 namespace: kserve-ci-e2e-test resourceVersion: "20773" uid: 6d3a13eb-dbaf-4d2f-bb90-d7d8f7c67b7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dce0b2786a namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 619de9c8-1112-46fc-961f-78b25875936f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dce0bf937c namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: ea369cb4-3418-4f14-8f20-d09f095b2496 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dce3cbac6a namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: b9755666-0543-406b-be78-dda8e69d5f22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T13:54:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:36Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dce44f8444 namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: 22d13fe3-2714-473e-a73b-0621272e873f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T13:54:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T14:03:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:54:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:53Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a900dd03c3d8b6 namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 0020e00c-df0a-45ec-8c8a-c74bb4f2bc8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a9015dcfcc2a03 namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: 7cfa3d2e-2247-4edb-9bc7-9f4aac0670de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a9015dcfccfdd8 namespace: kserve-ci-e2e-test resourceVersion: "25011" uid: 6c100a96-3dac-4a6e-8710-93bac60cda10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 5d1de448-78cb-4a0a-8927-5c71196199d6 kind: Event lastTimestamp: "2026-04-23T14:03:53Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T14:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:53Z" name: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb.18a9015e782d4494 namespace: kserve-ci-e2e-test resourceVersion: "25114" uid: b2cc6ef6-437e-4adf-8e5b-f32a09eff242 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8d150-predictor-5cd95d7749 namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 0206bad0-60e2-42be-8eb1-2755ea02a28b kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: 'Created pod: success-200-isvc-8d150-predictor-5cd95d7749-hpnmb' metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:35Z" name: success-200-isvc-8d150-predictor-5cd95d7749.18a900dc9ce55913 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: 0d315c41-e087-4ff8-b2db-884484305f2c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 21e5bc9a-a618-4f88-8396-80ab26d1c81a kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: Scaled up replica set success-200-isvc-8d150-predictor-5cd95d7749 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:35Z" name: success-200-isvc-8d150-predictor.18a900dc9ad9a7f2 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: b61754e4-1091-41a6-b677-b3cbdd8e4a6e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:54:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 5848c775-a951-4267-a2cf-d6b703c587ab kind: Event lastTimestamp: "2026-04-23T13:55:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:20Z" name: success-200-isvc-8d150-predictor.18a900e0195bfdeb namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: 5827c6bc-6b4b-4004-8cc3-edd815067c5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:54:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 5848c775-a951-4267-a2cf-d6b703c587ab kind: Event lastTimestamp: "2026-04-23T13:55:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:55:20Z" name: success-200-isvc-8d150-predictor.18a900e019bf10b5 namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 03ed634b-9568-42a7-bf57-a6b12c915f47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:55:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: 5848c775-a951-4267-a2cf-d6b703c587ab kind: Event lastTimestamp: "2026-04-23T13:56:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:56:05Z" name: success-200-isvc-8d150-predictor.18a900ea95d78c7f namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 7235af13-3dc2-41ef-a5cf-5cf40ec3a224 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:55:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d150-predictor namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: 5848c775-a951-4267-a2cf-d6b703c587ab kind: Event lastTimestamp: "2026-04-23T13:56:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:56:05Z" name: success-200-isvc-8d150-predictor.18a900ea96347926 namespace: kserve-ci-e2e-test resourceVersion: "21595" uid: b6b51f8a-8b2e-4613-a5dd-58aa8eeeece4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d150 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 1f6674f2-f6b7-4dfc-9506-de43e4cdc893 kind: Event lastTimestamp: "2026-04-23T13:55:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:55:23Z" name: success-200-isvc-8d150.18a900dc9ab9b623 namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: eb0abfc8-4d0a-45b9-8ae7-7a24be5f0d96 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d150 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 1f6674f2-f6b7-4dfc-9506-de43e4cdc893 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: 'Failed to update status for InferenceService "success-200-isvc-8d150": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8d150": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:35Z" name: success-200-isvc-8d150.18a900dc9b930637 namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: d206eaa4-31b6-4311-98ce-8eb49e54b994 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d150 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 1f6674f2-f6b7-4dfc-9506-de43e4cdc893 kind: Event lastTimestamp: "2026-04-23T13:54:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8d150": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:35Z" name: success-200-isvc-8d150.18a900dc9b9352bc namespace: kserve-ci-e2e-test resourceVersion: "20711" uid: 4bcfa494-5382-45ae-84f8-6ac2e7ba84b9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d150 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: 1f6674f2-f6b7-4dfc-9506-de43e4cdc893 kind: Event lastTimestamp: "2026-04-23T13:55:23Z" message: InferenceService [success-200-isvc-8d150] is Ready metadata: creationTimestamp: "2026-04-23T13:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:55:23Z" name: success-200-isvc-8d150.18a900e7beb5872b namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: 1a6ea7c2-5700-42f3-a385-1cfe9366c313 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:42:00.046868Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:42:00Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a9002cb869d232 namespace: kserve-ci-e2e-test resourceVersion: "13769" uid: 1443a55b-539c-4373-9c30-734b680d6b93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-94d25-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:00Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a9002cc5115090 namespace: kserve-ci-e2e-test resourceVersion: "13804" uid: db36c2ac-bbe0-4292-bb57-e30812907390 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13777" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:01Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:42:01Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a9002cf54f10cc namespace: kserve-ci-e2e-test resourceVersion: "13896" uid: fbbab65d-781f-4986-bbbb-4fab0c5a505a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:01Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" metadata: creationTimestamp: "2026-04-23T13:42:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:01Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a9002cf6a79e44 namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: 92961b0f-e698-44f0-a177-dfc589c7076a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:15Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" in 14.336s (14.336s including waiting). Image size: 1337604196 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:15Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a900304d2935d8 namespace: kserve-ci-e2e-test resourceVersion: "14084" uid: 5026d42f-ac19-479b-8638-5de8082a8029 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:15Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a9003050c2c293 namespace: kserve-ci-e2e-test resourceVersion: "14088" uid: e87a86de-9e77-4e02-b31b-c6cacb46d8ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:15Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a90030513a7c59 namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: 176d18dc-a064-4cd9-b7ea-6b62cc537666 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:15Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a9003051471291 namespace: kserve-ci-e2e-test resourceVersion: "14090" uid: 05f4095b-f35a-4ca3-8132-ae1aaae0eac1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:17Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.321s (2.321s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:17Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a90030dba18614 namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: 07d18245-a61e-4a0c-a2bb-fecc0b41f8c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:17Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a90030dfa140be namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 1d621b47-c1da-46c2-9ac1-cc3238cdb1b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:42:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:17Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a90030e0176f08 namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: e2b56959-edf3-4c7c-8c14-20dbb8a10d65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T13:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a9003121dac19d namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: 03592c9d-923f-4d22-89b6-df86dd7e110e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:34Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a90042b4b05ffb namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: c303bac7-5a19-4812-85e9-7068751291d9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:34Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a90042b4b14770 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 223df5fb-cd20-4d19-a875-1a39fda0f531 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9 namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: 74a8dd96-1bab-4a96-836c-3451008c1453 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9.18a90042d400bef0 namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 9bbd5da9-a451-4eb0-ba03-f755c35e6a77 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-94d25-predictor-6f676d4cb5 namespace: kserve-ci-e2e-test resourceVersion: "13756" uid: 76d7b454-d374-4673-b206-82941d9cbc32 kind: Event lastTimestamp: "2026-04-23T13:42:00Z" message: 'Created pod: success-200-isvc-94d25-predictor-6f676d4cb5-jjsc9' metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:00Z" name: success-200-isvc-94d25-predictor-6f676d4cb5.18a9002cb815886a namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: 61343519-3989-414a-9c17-fbac73164a4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:41:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "13755" uid: a5905fef-01b4-4895-9203-ff83a4cac8a4 kind: Event lastTimestamp: "2026-04-23T13:41:59Z" message: Scaled up replica set success-200-isvc-94d25-predictor-6f676d4cb5 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:41:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:41:59Z" name: success-200-isvc-94d25-predictor.18a9002cb2a97a9c namespace: kserve-ci-e2e-test resourceVersion: "13758" uid: 4d1aebfe-dd26-490b-b5cf-2c2aa4b08977 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "13770" uid: 95a0ab18-ed4b-470d-b27b-ddd69eb685a0 kind: Event lastTimestamp: "2026-04-23T13:42:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:45Z" name: success-200-isvc-94d25-predictor.18a90030381d1652 namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: b869f12e-7f13-469a-9399-9316c978b62e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "13770" uid: 95a0ab18-ed4b-470d-b27b-ddd69eb685a0 kind: Event lastTimestamp: "2026-04-23T13:42:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:45Z" name: success-200-isvc-94d25-predictor.18a90030385eb2cc namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: 7d559933-b625-4742-8ea3-50a446c5021f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:43:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: 95a0ab18-ed4b-470d-b27b-ddd69eb685a0 kind: Event lastTimestamp: "2026-04-23T13:43:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:30Z" name: success-200-isvc-94d25-predictor.18a9003ab50ec64a namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: fcacce2d-56d9-4c37-b824-78bc5358ca19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:43:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-94d25-predictor namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: 95a0ab18-ed4b-470d-b27b-ddd69eb685a0 kind: Event lastTimestamp: "2026-04-23T13:43:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:30Z" name: success-200-isvc-94d25-predictor.18a9003ab5711eca namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: 4bc65d7d-6d5a-4f5e-a8b4-bf56c451a7c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:42:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-94d25 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: dec66086-b5e7-43a3-8706-a4226eda5f97 kind: Event lastTimestamp: "2026-04-23T13:43:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:42:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:05Z" name: success-200-isvc-94d25.18a9002cb8d00de6 namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: c6a4f82b-6def-4da4-9c98-be52825218ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-94d25 namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: dec66086-b5e7-43a3-8706-a4226eda5f97 kind: Event lastTimestamp: "2026-04-23T13:43:05Z" message: InferenceService [success-200-isvc-94d25] is Ready metadata: creationTimestamp: "2026-04-23T13:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:05Z" name: success-200-isvc-94d25.18a9003bdcb9dc6d namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: 31ae7391-0304-4389-9b19-89e676c6ee1e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T14:04:25.013442Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25506" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T14:04:25Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a90165deab1813 namespace: kserve-ci-e2e-test resourceVersion: "25511" uid: 37775d39-035b-40a8-b506-de0e6e7412f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a8c3e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:25Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a90165eb8aa3a2 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: 4af945e8-c609-498c-b36d-b00d364082f4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:04:26Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T14:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T14:04:26Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a901661beb5675 namespace: kserve-ci-e2e-test resourceVersion: "25567" uid: a0f41115-1889-42f6-946d-84e0e923bca7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:04:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T14:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:26Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a901661d6cf1bc namespace: kserve-ci-e2e-test resourceVersion: "25572" uid: 2f2be701-203a-4eaf-b7ae-c5574bd92aa3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:04:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T14:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:26Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a9016620d5bb0f namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: aa41da08-bf0f-4c3c-b456-b13bcfa17995 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:04:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T14:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:26Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a90166214c4e35 namespace: kserve-ci-e2e-test resourceVersion: "25574" uid: 5a4e6922-5c62-4ea6-8c9f-b536aea13e9c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:04:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T14:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:26Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a90166215a61a5 namespace: kserve-ci-e2e-test resourceVersion: "25575" uid: e43607d2-88ac-4acd-845f-47c373222cb1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:04:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T14:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:26Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a9016624491734 namespace: kserve-ci-e2e-test resourceVersion: "25576" uid: ab3b0d99-374a-44d6-a2ed-95d31be812c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:04:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:26Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a9016624bcee6c namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: ac301aef-c4d5-4a05-b084-51cf4b8d3253 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:04:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:05:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T14:04:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:05:03Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a901668deba64f namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: e691c6c7-eb15-451d-90cf-b7a993a804ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:13:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:13:39Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a901e70c6751ca namespace: kserve-ci-e2e-test resourceVersion: "29925" uid: c5687917-1576-4400-9b57-ad81cdaf9e36 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: c88719ce-0c49-469d-b128-da335ca5a48e kind: Event lastTimestamp: "2026-04-23T14:13:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:13:39Z" name: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr.18a901e70c67c48b namespace: kserve-ci-e2e-test resourceVersion: "29927" uid: 7d1ca4e1-413a-4f52-825b-2067fab717b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a8c3e-predictor-7f68d969df namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: 8f8fddae-d731-4c1f-b00d-69c968734ddc kind: Event lastTimestamp: "2026-04-23T14:04:25Z" message: 'Created pod: success-200-isvc-a8c3e-predictor-7f68d969df-qqgdr' metadata: creationTimestamp: "2026-04-23T14:04:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:25Z" name: success-200-isvc-a8c3e-predictor-7f68d969df.18a90165de089d1d namespace: kserve-ci-e2e-test resourceVersion: "25507" uid: 87c178e2-ed49-4bbb-a3c1-0f5a0718c358 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25483" uid: e3e8e91e-ea76-4bcc-95a2-f1d9495cd813 kind: Event lastTimestamp: "2026-04-23T14:04:24Z" message: Scaled up replica set success-200-isvc-a8c3e-predictor-7f68d969df from 0 to 1 metadata: creationTimestamp: "2026-04-23T14:04:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:24Z" name: success-200-isvc-a8c3e-predictor.18a90165db255832 namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 0f61ddb7-5f63-4c7e-aa8f-7a122745b456 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:04:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: 1400e830-6075-49b1-be97-a5a86408ab5f kind: Event lastTimestamp: "2026-04-23T14:04:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:54Z" name: success-200-isvc-a8c3e-predictor.18a9016959fab221 namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: c85ab112-6b6a-4103-8304-ee3efb9e98ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:04:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: 1400e830-6075-49b1-be97-a5a86408ab5f kind: Event lastTimestamp: "2026-04-23T14:04:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:54Z" name: success-200-isvc-a8c3e-predictor.18a901695a601846 namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 13400ec2-1d07-492f-b4be-0b01f8c493ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:05:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: 1400e830-6075-49b1-be97-a5a86408ab5f kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:25Z" name: success-200-isvc-a8c3e-predictor.18a90170586de773 namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: bbafb41e-d416-4c3f-8a84-59bc1bd1a598 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:05:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a8c3e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: 1400e830-6075-49b1-be97-a5a86408ab5f kind: Event lastTimestamp: "2026-04-23T14:05:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:25Z" name: success-200-isvc-a8c3e-predictor.18a90170591c503e namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 276122a7-b04a-4e63-a2aa-4866217a39f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:04:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "25460" uid: c0c08794-3438-45c9-8455-bc8eb5965108 kind: Event lastTimestamp: "2026-04-23T14:05:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T14:04:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:05:14Z" name: success-200-isvc-a8c3e.18a90165db0e04fa namespace: kserve-ci-e2e-test resourceVersion: "26344" uid: d80059e4-334c-4386-a613-30b4cb7d254b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "25460" uid: c0c08794-3438-45c9-8455-bc8eb5965108 kind: Event lastTimestamp: "2026-04-23T14:04:24Z" message: 'Failed to update status for InferenceService "success-200-isvc-a8c3e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a8c3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:04:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:24Z" name: success-200-isvc-a8c3e.18a90165dbf2abe4 namespace: kserve-ci-e2e-test resourceVersion: "25499" uid: 85b113ee-6a0a-4650-9e40-3188da71e529 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "25460" uid: c0c08794-3438-45c9-8455-bc8eb5965108 kind: Event lastTimestamp: "2026-04-23T14:04:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a8c3e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:04:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:24Z" name: success-200-isvc-a8c3e.18a90165dbf2ffcd namespace: kserve-ci-e2e-test resourceVersion: "25502" uid: 7eea61a0-a0a8-4c97-80c4-8554cfe033bd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:05:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a8c3e namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: c0c08794-3438-45c9-8455-bc8eb5965108 kind: Event lastTimestamp: "2026-04-23T14:05:14Z" message: InferenceService [success-200-isvc-a8c3e] is Ready metadata: creationTimestamp: "2026-04-23T14:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:05:14Z" name: success-200-isvc-a8c3e.18a9017149282152 namespace: kserve-ci-e2e-test resourceVersion: "26343" uid: 92fb57c0-432f-4972-a8a5-28e2537ffceb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:43:34.515358Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:43:34Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a90042b72c9c81 namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 4ac7ed39-b4a4-4d32-ab25-75bf3f811e9c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14933" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:43:34Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a90042d147856c namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: 111d3a32-b8f7-43e7-a8da-d967992a6c23 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:34Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a90042d28d1c4f namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 31c4b432-e02b-4fcd-824b-eb521206f500 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a90042d56a4093 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: c0c3a9fd-586d-4786-bcd2-d124c9393b76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a90042d5da84d0 namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: a32e1ea1-1661-4ab1-b380-170774074260 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a90042d5e8cf29 namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: ac0e7ebe-aa20-40e4-8d8f-4b3170f7aa3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a90042d90a636c namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 38a62391-9a6b-4add-b8cf-dd5b3e489cf3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:43:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:35Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a90042d9917f28 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 8104eb99-03d5-4523-9357-6a774e62fa14 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T13:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:52:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:52Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a900431ded0cf8 namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: 2a684bfc-f911-4e1f-8915-17092a4dba03 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:49Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a900c3e96934d2 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: caa57450-dbcc-465b-92f5-c5d0bbd3f4ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:49Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a900c3e96985ff namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 1f09c767-59d7-4f3f-9741-30dd702ae574 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 33060c2f-cf52-4fb3-a6d2-b785274a7767 kind: Event lastTimestamp: "2026-04-23T13:52:52Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:52Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2.18a900c4924c8c29 namespace: kserve-ci-e2e-test resourceVersion: "19424" uid: 53ea5251-3ead-4a32-80b3-e3ce735655b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d2d70-predictor-5cd95cc7dc namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: ebe71ce8-b51c-4824-adf7-7f1ac0638208 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: 'Created pod: success-200-isvc-d2d70-predictor-5cd95cc7dc-9qgt2' metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:34Z" name: success-200-isvc-d2d70-predictor-5cd95cc7dc.18a90042b6c8d161 namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 784f9ce7-44a8-475b-8b7b-df86092781cd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: dedc6847-8b3d-48e1-acab-08e0449b7864 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Scaled up replica set success-200-isvc-d2d70-predictor-5cd95cc7dc from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:34Z" name: success-200-isvc-d2d70-predictor.18a90042b3cdb0d0 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 8216335d-f8c6-4294-be7b-89a3a5f6cbf9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:43:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: 302507b0-3064-4b16-8438-386baf138e56 kind: Event lastTimestamp: "2026-04-23T13:44:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:04Z" name: success-200-isvc-d2d70-predictor.18a900463247c22a namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 046c210a-2fff-4f94-9a4d-60a36d2c88c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:43:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: 302507b0-3064-4b16-8438-386baf138e56 kind: Event lastTimestamp: "2026-04-23T13:44:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:04Z" name: success-200-isvc-d2d70-predictor.18a9004632aa1ea5 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 8aa824ba-8b11-4d0e-811b-5ee6135b0e4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 302507b0-3064-4b16-8438-386baf138e56 kind: Event lastTimestamp: "2026-04-23T13:44:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:49Z" name: success-200-isvc-d2d70-predictor.18a9004d30061c3a namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 5b26a3e1-3246-4a8b-a2cc-fa5b7e3d722c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2d70-predictor namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 302507b0-3064-4b16-8438-386baf138e56 kind: Event lastTimestamp: "2026-04-23T13:44:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:49Z" name: success-200-isvc-d2d70-predictor.18a9004d3072439b namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 3b86791a-2c12-4c4f-a54c-9fe81a536752 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: f4a499cd-722a-4f66-b057-b3f60a7f46d5 kind: Event lastTimestamp: "2026-04-23T13:44:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:22Z" name: success-200-isvc-d2d70.18a90042b3a56ac1 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 62745d21-658b-4f31-9123-b370a2031a12 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: f4a499cd-722a-4f66-b057-b3f60a7f46d5 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: 'Failed to update status for InferenceService "success-200-isvc-d2d70": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d2d70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:34Z" name: success-200-isvc-d2d70.18a90042b40fd5cc namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: b2501889-87da-475b-8a3d-1138bc968947 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: f4a499cd-722a-4f66-b057-b3f60a7f46d5 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d2d70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:34Z" name: success-200-isvc-d2d70.18a90042b410263e namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: accc6b90-c307-43fe-b874-674e88b6ec40 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: f4a499cd-722a-4f66-b057-b3f60a7f46d5 kind: Event lastTimestamp: "2026-04-23T13:44:22Z" message: InferenceService [success-200-isvc-d2d70] is Ready metadata: creationTimestamp: "2026-04-23T13:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:22Z" name: success-200-isvc-d2d70.18a9004dd8b860c1 namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: 5c568c6b-800f-472e-be6b-1bf492a59c6c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:44:10.386790Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f7bdd-predictor-54969f468c-dq4nt to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:44:10Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004b11473a27 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 241c7bdb-25c5-4d07-aabe-75bdabf53b3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f7bdd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:10Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004b1a514a40 namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: ee6ef372-cab9-4cd0-b147-1d22ce26e517 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:44:11Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:44:11Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004b4e5dfdca namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 31534327-7dfb-4e1f-a9f5-5d30ac39ad9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:44:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:11Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004b4fbe2846 namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: c086e26a-59f0-46ab-a303-d64041183980 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:44:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:11Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004b52498c25 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 9b799d38-2860-4a0c-88f1-02075f0482ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:44:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:11Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004b52ac3619 namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: 103ef5ff-4f76-4c8c-87bd-1335da165ad8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:44:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:11Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004b52b8ba81 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: eb83b5e0-5fca-4491-b7c3-03c25595bffd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:44:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:11Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004b561bd550 namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: 6a8b5910-da4d-4a84-902d-0d2727a86cc6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:44:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:11Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004b56824caf namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: c1923582-13bf-4397-91a9-bcb108953a4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:44:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:44:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:49Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a9004bc24a9533 namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: ae050ce2-fbfd-49da-9c26-6dd2e3c4a672 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a900cc40fee2db namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: cb69f13f-f9bb-4f58-a416-f391f7ef15b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: feee8d88-99ee-4507-8190-0cb66713e21d kind: Event lastTimestamp: "2026-04-23T13:53:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:25Z" name: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt.18a900cc40ff67dd namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: 5e8a52ca-1233-48ad-9746-aa1984925631 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f7bdd-predictor-54969f468c namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 8dcae20b-5ba4-4ad6-98c8-565a0ba70f90 kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: 'Created pod: success-200-isvc-f7bdd-predictor-54969f468c-dq4nt' metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:10Z" name: success-200-isvc-f7bdd-predictor-54969f468c.18a9004b10b8ecf4 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 4ff4f4f5-efcf-425d-bf2d-3c9e5b280a59 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 1cc27be9-0ee6-4c76-9095-df382429021a kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: Scaled up replica set success-200-isvc-f7bdd-predictor-54969f468c from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:10Z" name: success-200-isvc-f7bdd-predictor.18a9004b0f17147d namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: eaafedbb-f458-4da8-95bb-3968a04ab267 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 96f66bf0-e559-4750-a476-33cb693e91ae kind: Event lastTimestamp: "2026-04-23T13:45:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:25Z" name: success-200-isvc-f7bdd-predictor.18a9004e8dba44eb namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: b897730e-9b82-4281-994a-544dff7bb9e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:44:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 96f66bf0-e559-4750-a476-33cb693e91ae kind: Event lastTimestamp: "2026-04-23T13:45:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:25Z" name: success-200-isvc-f7bdd-predictor.18a9004e8e1a93ce namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: bb123c6c-515f-4c05-8b95-95198be6c6dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 96f66bf0-e559-4750-a476-33cb693e91ae kind: Event lastTimestamp: "2026-04-23T13:44:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:40Z" name: success-200-isvc-f7bdd-predictor.18a900520d0dbc8a namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: f1c94e7e-bf7a-4491-a1f4-b87fa8083b5c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 96f66bf0-e559-4750-a476-33cb693e91ae kind: Event lastTimestamp: "2026-04-23T13:44:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:40Z" name: success-200-isvc-f7bdd-predictor.18a900520d71f348 namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 8d40a0d0-3450-4180-8d26-4cbeec330679 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: b74e380c-c303-4979-b403-53089debee3a kind: Event lastTimestamp: "2026-04-23T13:44:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:59Z" name: success-200-isvc-f7bdd.18a9004b0f1f4859 namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: 933bc874-0801-47a1-bf8e-f85e71d4fac9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: b74e380c-c303-4979-b403-53089debee3a kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-f7bdd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f7bdd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:10Z" name: success-200-isvc-f7bdd.18a9004b0f96c88a namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: 443ea5da-0e27-44c6-8928-bd0b4e7b933b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: b74e380c-c303-4979-b403-53089debee3a kind: Event lastTimestamp: "2026-04-23T13:44:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f7bdd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:44:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:10Z" name: success-200-isvc-f7bdd.18a9004b0f97217a namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: 654abbae-3aaf-4be3-a6f4-2171f1a044ef reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7bdd namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: b74e380c-c303-4979-b403-53089debee3a kind: Event lastTimestamp: "2026-04-23T13:44:59Z" message: InferenceService [success-200-isvc-f7bdd] is Ready metadata: creationTimestamp: "2026-04-23T13:44:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:59Z" name: success-200-isvc-f7bdd.18a900567cf957a6 namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: ef7a1e2f-3936-46e2-9542-5d20e3e0da2b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T14:03:50.446280Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-feffa-predictor-5cff45889-498mt to ip-10-0-141-176.ec2.internal metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9015dd24e0ec8 namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: ab99bfcc-2ff1-46b4-b617-758ea213c8fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25023" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9015debd13aee namespace: kserve-ci-e2e-test resourceVersion: "25072" uid: b4cd1757-d364-4cc9-8759-4ee0261b2679 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9015ded46a9b1 namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 791b04a1-69ed-4ccc-838d-0d9306ab6e7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9015df039f582 namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: 2d8ebcbd-0d90-4512-a1e0-3eaefbf47650 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9015df0aa83f8 namespace: kserve-ci-e2e-test resourceVersion: "25076" uid: 7f69086d-9ed5-4649-a556-d867d0ba7396 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9015df0b69de4 namespace: kserve-ci-e2e-test resourceVersion: "25077" uid: 8941650f-a4fc-4130-88b2-31e235e28d11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:03:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:51Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9015df3e2ffde namespace: kserve-ci-e2e-test resourceVersion: "25078" uid: a21221d8-d369-4d61-963e-cc2e218e24fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:03:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:03:51Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9015df45c9435 namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: 3349126f-66e6-4408-8d2d-1b6deca9830e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T14:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:04:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T14:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:28Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9015e612b2446 namespace: kserve-ci-e2e-test resourceVersion: "25609" uid: ca7c42d6-f55e-4d14-9cdb-fe87c4823cb7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:21:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T14:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:21:09Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9024fd96ff254 namespace: kserve-ci-e2e-test resourceVersion: "33060" uid: 8ec48248-c024-414a-8aaf-b15aa92490e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-feffa-predictor-5cff45889-498mt namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 6e0ec7bd-e53b-4dbf-923e-d7303fdf8db4 kind: Event lastTimestamp: "2026-04-23T14:21:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T14:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:21:09Z" name: success-200-isvc-feffa-predictor-5cff45889-498mt.18a9024fd970911c namespace: kserve-ci-e2e-test resourceVersion: "33061" uid: 5fbf04e4-5049-4155-9929-f27dc8384c1e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-176.ec2.internal source: component: kubelet host: ip-10-0-141-176.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-feffa-predictor-5cff45889 namespace: kserve-ci-e2e-test resourceVersion: "24996" uid: 24a42bab-39f6-457b-a315-fb5d82755512 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: 'Created pod: success-200-isvc-feffa-predictor-5cff45889-498mt' metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa-predictor-5cff45889.18a9015dd1bf527b namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: 90b471ac-fc56-4294-b225-3d757f88f59b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: f73aaec6-9e24-40f2-9aa8-e2f0f70ee530 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: Scaled up replica set success-200-isvc-feffa-predictor-5cff45889 from 0 to 1 metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa-predictor.18a9015dce6d66b0 namespace: kserve-ci-e2e-test resourceVersion: "25001" uid: 6687f632-cf91-4d16-818c-c604edcc194b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:04:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "24995" uid: faef6bea-519d-4940-ac6b-d686e999ef42 kind: Event lastTimestamp: "2026-04-23T14:04:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:35Z" name: success-200-isvc-feffa-predictor.18a901614d07ab22 namespace: kserve-ci-e2e-test resourceVersion: "25665" uid: 4f28b4a3-a7ba-4ad1-b74c-95740f6192ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T14:04:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "24995" uid: faef6bea-519d-4940-ac6b-d686e999ef42 kind: Event lastTimestamp: "2026-04-23T14:04:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:35Z" name: success-200-isvc-feffa-predictor.18a901614d6a19c8 namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: 5b8ecaff-53bd-4cd1-9bf1-9a795a3d6f5a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T14:04:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25238" uid: faef6bea-519d-4940-ac6b-d686e999ef42 kind: Event lastTimestamp: "2026-04-23T14:05:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:05Z" name: success-200-isvc-feffa-predictor.18a90164cc0bfa34 namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: e354c4ac-9b96-409f-9c06-13da7bae42d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T14:04:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-feffa-predictor namespace: kserve-ci-e2e-test resourceVersion: "25238" uid: faef6bea-519d-4940-ac6b-d686e999ef42 kind: Event lastTimestamp: "2026-04-23T14:05:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T14:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:05Z" name: success-200-isvc-feffa-predictor.18a90164cc69a025 namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: 817e7843-8c4c-4256-bd76-ade13fb41289 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-feffa namespace: kserve-ci-e2e-test resourceVersion: "24977" uid: ab51a8fe-904a-4573-9cb7-66f695aa28c3 kind: Event lastTimestamp: "2026-04-23T14:04:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:38Z" name: success-200-isvc-feffa.18a9015dce3a26c1 namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 07fc8f66-7a8d-4dcc-ae38-6f6a272aeee7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-feffa namespace: kserve-ci-e2e-test resourceVersion: "24977" uid: ab51a8fe-904a-4573-9cb7-66f695aa28c3 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-feffa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-feffa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa.18a9015dcf236098 namespace: kserve-ci-e2e-test resourceVersion: "25006" uid: 598824b0-67f4-4235-89f5-b6f5da20ce2e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-feffa namespace: kserve-ci-e2e-test resourceVersion: "24977" uid: ab51a8fe-904a-4573-9cb7-66f695aa28c3 kind: Event lastTimestamp: "2026-04-23T14:03:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-feffa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:03:50Z" name: success-200-isvc-feffa.18a9015dcf23b8a8 namespace: kserve-ci-e2e-test resourceVersion: "25010" uid: 6e3900c8-4db6-4a7e-885c-dcf4458ba3a3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-feffa namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: ab51a8fe-904a-4573-9cb7-66f695aa28c3 kind: Event lastTimestamp: "2026-04-23T14:04:38Z" message: InferenceService [success-200-isvc-feffa] is Ready metadata: creationTimestamp: "2026-04-23T14:04:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:38Z" name: success-200-isvc-feffa.18a901691c7d7755 namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: 9426e3ca-cc8c-4531-96a0-56451faba8b6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:43:20.140689Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-94d25-5664f9bbb4-v4vxx namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: 0f72de3a-e751-4e2d-b589-a4cac2dd1dd1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-94d25-5664f9bbb4-v4vxx to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:43:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:43:20Z" name: switch-graph-94d25-5664f9bbb4-v4vxx.18a9003f5e60837f namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 59617e90-daf1-4ba7-8790-a0f922747fff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-94d25-5664f9bbb4-v4vxx namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 0f72de3a-e751-4e2d-b589-a4cac2dd1dd1 kind: Event lastTimestamp: "2026-04-23T13:43:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-94d25-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:20Z" name: switch-graph-94d25-5664f9bbb4-v4vxx.18a9003f690d5bfb namespace: kserve-ci-e2e-test resourceVersion: "14717" uid: 33c1a928-4727-49da-84c5-0d438f4658ac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-94d25-5664f9bbb4-v4vxx namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 0f72de3a-e751-4e2d-b589-a4cac2dd1dd1 kind: Event lastTimestamp: "2026-04-23T13:43:21Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:43:21Z" name: switch-graph-94d25-5664f9bbb4-v4vxx.18a9003f9ac09cfa namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: a7c5a262-7ceb-4573-88e0-3d83107d461e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-94d25} kind: Pod name: switch-graph-94d25-5664f9bbb4-v4vxx namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 0f72de3a-e751-4e2d-b589-a4cac2dd1dd1 kind: Event lastTimestamp: "2026-04-23T13:43:21Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" metadata: creationTimestamp: "2026-04-23T13:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:21Z" name: switch-graph-94d25-5664f9bbb4-v4vxx.18a9003f9c1558f4 namespace: kserve-ci-e2e-test resourceVersion: "14727" uid: 1a5d0d65-a9aa-41fd-8701-7477ae64712e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-94d25} kind: Pod name: switch-graph-94d25-5664f9bbb4-v4vxx namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 0f72de3a-e751-4e2d-b589-a4cac2dd1dd1 kind: Event lastTimestamp: "2026-04-23T13:43:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" in 2.581s (2.581s including waiting). Image size: 216346181 bytes.' metadata: creationTimestamp: "2026-04-23T13:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:23Z" name: switch-graph-94d25-5664f9bbb4-v4vxx.18a9004035f0699b namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 94812644-21f1-4104-9fd0-f6b0befd87de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-94d25} kind: Pod name: switch-graph-94d25-5664f9bbb4-v4vxx namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 0f72de3a-e751-4e2d-b589-a4cac2dd1dd1 kind: Event lastTimestamp: "2026-04-23T13:43:23Z" message: 'Created container: switch-graph-94d25' metadata: creationTimestamp: "2026-04-23T13:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:23Z" name: switch-graph-94d25-5664f9bbb4-v4vxx.18a9004038d4d7ab namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: 14a4d689-da87-4849-a931-e80e4bf9940b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-94d25} kind: Pod name: switch-graph-94d25-5664f9bbb4-v4vxx namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 0f72de3a-e751-4e2d-b589-a4cac2dd1dd1 kind: Event lastTimestamp: "2026-04-23T13:43:23Z" message: Started container switch-graph-94d25 metadata: creationTimestamp: "2026-04-23T13:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:23Z" name: switch-graph-94d25-5664f9bbb4-v4vxx.18a900403933aba4 namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: 60af3640-c676-4937-9dd0-57f6a96343dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-94d25} kind: Pod name: switch-graph-94d25-5664f9bbb4-v4vxx namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 0f72de3a-e751-4e2d-b589-a4cac2dd1dd1 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Stopping container switch-graph-94d25 metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:34Z" name: switch-graph-94d25-5664f9bbb4-v4vxx.18a90042accc01c0 namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: 1c2f617e-b99f-4884-96e6-679f75a67a21 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-94d25} kind: Pod name: switch-graph-94d25-5664f9bbb4-v4vxx namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 0f72de3a-e751-4e2d-b589-a4cac2dd1dd1 kind: Event lastTimestamp: "2026-04-23T13:44:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:00Z" name: switch-graph-94d25-5664f9bbb4-v4vxx.18a90042edf5cb9c namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 95111672-e54d-4b7a-b2da-32b00dc8124c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-94d25-5664f9bbb4 namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: 383fdac0-f979-4349-a0dc-bed11be3d9d3 kind: Event lastTimestamp: "2026-04-23T13:43:20Z" message: 'Created pod: switch-graph-94d25-5664f9bbb4-v4vxx' metadata: creationTimestamp: "2026-04-23T13:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:20Z" name: switch-graph-94d25-5664f9bbb4.18a9003f5dee77ee namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 431db9d8-edc2-4a89-920b-c1c432a79222 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-94d25 namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 91a0b0fd-20e5-4bd1-b179-cc91807af9b8 kind: Event lastTimestamp: "2026-04-23T13:43:20Z" message: Scaled up replica set switch-graph-94d25-5664f9bbb4 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:20Z" name: switch-graph-94d25.18a9003f5d34f36f namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 3741431e-eb8b-4ed3-a06f-b833a2c8007b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-94d25 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: 3f44a0a6-520e-4bcc-8b38-6ddf3e4e029d kind: Event lastTimestamp: "2026-04-23T13:43:20Z" message: InferenceGraph [switch-graph-94d25] is Ready metadata: creationTimestamp: "2026-04-23T13:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:20Z" name: switch-graph-94d25.18a9003f610239fc namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: f2e1df04-b359-4104-8a57-953e67b6d01a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:44:34.553721Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d2d70-6d98679489-grd7b namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 43db906c-91ff-4eac-84ed-6ee746033d17 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d2d70-6d98679489-grd7b to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T13:44:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:44:34Z" name: switch-graph-d2d70-6d98679489-grd7b.18a90050b1bd49e7 namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 866e4ea4-ff4b-42fb-ae86-25a4bc1fee55 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:34Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d2d70-6d98679489-grd7b namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 43db906c-91ff-4eac-84ed-6ee746033d17 kind: Event lastTimestamp: "2026-04-23T13:44:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d2d70-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:34Z" name: switch-graph-d2d70-6d98679489-grd7b.18a90050bdce9984 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 31364960-f013-450c-8604-135bfa7d6311 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:35Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d2d70-6d98679489-grd7b namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 43db906c-91ff-4eac-84ed-6ee746033d17 kind: Event lastTimestamp: "2026-04-23T13:44:35Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:44:35Z" name: switch-graph-d2d70-6d98679489-grd7b.18a90050ee4e8b72 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 446233cf-97c4-4e5a-97e9-70a52decd1ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2d70} kind: Pod name: switch-graph-d2d70-6d98679489-grd7b namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 43db906c-91ff-4eac-84ed-6ee746033d17 kind: Event lastTimestamp: "2026-04-23T13:44:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T13:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:35Z" name: switch-graph-d2d70-6d98679489-grd7b.18a90050ef7d74d5 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 6c4ae88e-5b14-4cfb-b728-3941604c3447 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2d70} kind: Pod name: switch-graph-d2d70-6d98679489-grd7b namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 43db906c-91ff-4eac-84ed-6ee746033d17 kind: Event lastTimestamp: "2026-04-23T13:44:35Z" message: 'Created container: switch-graph-d2d70' metadata: creationTimestamp: "2026-04-23T13:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:35Z" name: switch-graph-d2d70-6d98679489-grd7b.18a90050f25156d2 namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: e68a20c5-cd38-4f02-9e78-84fa3bbf5d90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2d70} kind: Pod name: switch-graph-d2d70-6d98679489-grd7b namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 43db906c-91ff-4eac-84ed-6ee746033d17 kind: Event lastTimestamp: "2026-04-23T13:44:35Z" message: Started container switch-graph-d2d70 metadata: creationTimestamp: "2026-04-23T13:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:35Z" name: switch-graph-d2d70-6d98679489-grd7b.18a90050f2cd370c namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: 03568d2b-cdca-493c-a0d6-edf802c73951 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2d70} kind: Pod name: switch-graph-d2d70-6d98679489-grd7b namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 43db906c-91ff-4eac-84ed-6ee746033d17 kind: Event lastTimestamp: "2026-04-23T13:52:49Z" message: Stopping container switch-graph-d2d70 metadata: creationTimestamp: "2026-04-23T13:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:49Z" name: switch-graph-d2d70-6d98679489-grd7b.18a900c3e379bd6e namespace: kserve-ci-e2e-test resourceVersion: "19282" uid: 49a78d07-506d-4d96-8870-058be8507e92 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2d70} kind: Pod name: switch-graph-d2d70-6d98679489-grd7b namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 43db906c-91ff-4eac-84ed-6ee746033d17 kind: Event lastTimestamp: "2026-04-23T13:53:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:17Z" name: switch-graph-d2d70-6d98679489-grd7b.18a900c4aacd95d4 namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: ffd097e3-e3ec-4f71-86bd-86a3335ae3ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d2d70-6d98679489 namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: cdf14a9e-6e88-4a45-ac4e-15a52a33f2d0 kind: Event lastTimestamp: "2026-04-23T13:44:34Z" message: 'Created pod: switch-graph-d2d70-6d98679489-grd7b' metadata: creationTimestamp: "2026-04-23T13:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:34Z" name: switch-graph-d2d70-6d98679489.18a90050b1669431 namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 2416b100-8eaa-404a-8ab2-130d25f26119 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 2c694165-da22-4717-977e-58e32ccf8e9b kind: Event lastTimestamp: "2026-04-23T13:44:34Z" message: Scaled up replica set switch-graph-d2d70-6d98679489 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:34Z" name: switch-graph-d2d70.18a90050b0af389a namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: f54c0711-f5be-4bdf-9835-f42c3cb968a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: c5899e49-5fda-46cc-b3bb-382c6ca56e36 kind: Event lastTimestamp: "2026-04-23T13:44:34Z" message: InferenceGraph [switch-graph-d2d70] is Ready metadata: creationTimestamp: "2026-04-23T13:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:44:34Z" name: switch-graph-d2d70.18a90050b3ee2f41 namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: da7ddbd8-c376-437a-bbad-4a62d4047181 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: 41961767-1648-42eb-93bc-5eb0db5f40af kind: Event lastTimestamp: "2026-04-23T13:44:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:49Z" name: switch-graph-d2d70.18a900542f059896 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: b770e76a-8d45-4c6e-ac1e-907b7361258c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:44:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: 41961767-1648-42eb-93bc-5eb0db5f40af kind: Event lastTimestamp: "2026-04-23T13:44:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:49Z" name: switch-graph-d2d70.18a900542f63dee2 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 7e2455fc-29a3-43be-8863-1c3f08d42f75 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 41961767-1648-42eb-93bc-5eb0db5f40af kind: Event lastTimestamp: "2026-04-23T13:45:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:04Z" name: switch-graph-d2d70.18a90057adefffee namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: 17c756a3-18a7-4c5f-84ac-a709f2ef544d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d2d70 namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 41961767-1648-42eb-93bc-5eb0db5f40af kind: Event lastTimestamp: "2026-04-23T13:45:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:04Z" name: switch-graph-d2d70.18a90057ae4a95eb namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 66f1b725-9947-4e0c-aa19-1c12782ad972 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T14:04:50.446139Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-feffa-7cd874cdb5-n9vxb namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: 361259e9-32f4-43ac-bfe5-7d4ae4337229 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-feffa-7cd874cdb5-n9vxb to ip-10-0-132-207.ec2.internal metadata: creationTimestamp: "2026-04-23T14:04:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T14:04:50Z" name: switch-graph-feffa-7cd874cdb5-n9vxb.18a9016bca9332f8 namespace: kserve-ci-e2e-test resourceVersion: "25824" uid: 582dab5c-c9bc-47f1-ba6e-9a9f590a91be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6d8877df8-p5w5s source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-feffa-7cd874cdb5-n9vxb namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 361259e9-32f4-43ac-bfe5-7d4ae4337229 kind: Event lastTimestamp: "2026-04-23T14:04:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-feffa-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:50Z" name: switch-graph-feffa-7cd874cdb5-n9vxb.18a9016bd2bed744 namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: 9ab0aed5-1a51-4bda-8dea-e7132fb5e2c8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-feffa-7cd874cdb5-n9vxb namespace: kserve-ci-e2e-test resourceVersion: "25829" uid: 361259e9-32f4-43ac-bfe5-7d4ae4337229 kind: Event lastTimestamp: "2026-04-23T14:04:51Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T14:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T14:04:51Z" name: switch-graph-feffa-7cd874cdb5-n9vxb.18a9016c0717a8ec namespace: kserve-ci-e2e-test resourceVersion: "25846" uid: b735cbc9-e44c-43c3-a0e5-614fe98f9f90 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-feffa} kind: Pod name: switch-graph-feffa-7cd874cdb5-n9vxb namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 361259e9-32f4-43ac-bfe5-7d4ae4337229 kind: Event lastTimestamp: "2026-04-23T14:04:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T14:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:51Z" name: switch-graph-feffa-7cd874cdb5-n9vxb.18a9016c083d9f69 namespace: kserve-ci-e2e-test resourceVersion: "25849" uid: 8f51e25e-4dcd-49ff-a89e-ef734a721506 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-feffa} kind: Pod name: switch-graph-feffa-7cd874cdb5-n9vxb namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 361259e9-32f4-43ac-bfe5-7d4ae4337229 kind: Event lastTimestamp: "2026-04-23T14:04:51Z" message: 'Created container: switch-graph-feffa' metadata: creationTimestamp: "2026-04-23T14:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:51Z" name: switch-graph-feffa-7cd874cdb5-n9vxb.18a9016c0acf1296 namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: 2b8805d3-e2b6-457c-b047-c2598ce46a26 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-feffa} kind: Pod name: switch-graph-feffa-7cd874cdb5-n9vxb namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 361259e9-32f4-43ac-bfe5-7d4ae4337229 kind: Event lastTimestamp: "2026-04-23T14:04:51Z" message: Started container switch-graph-feffa metadata: creationTimestamp: "2026-04-23T14:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:04:51Z" name: switch-graph-feffa-7cd874cdb5-n9vxb.18a9016c0b3b4cc1 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: a9b5abf7-71a9-442b-9220-b392a72806fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-feffa} kind: Pod name: switch-graph-feffa-7cd874cdb5-n9vxb namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 361259e9-32f4-43ac-bfe5-7d4ae4337229 kind: Event lastTimestamp: "2026-04-23T14:21:09Z" message: Stopping container switch-graph-feffa metadata: creationTimestamp: "2026-04-23T14:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:21:09Z" name: switch-graph-feffa-7cd874cdb5-n9vxb.18a9024fd2066169 namespace: kserve-ci-e2e-test resourceVersion: "33045" uid: 3483f948-1e8b-41b2-ac3b-b1ed313a5210 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T14:21:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-feffa} kind: Pod name: switch-graph-feffa-7cd874cdb5-n9vxb namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 361259e9-32f4-43ac-bfe5-7d4ae4337229 kind: Event lastTimestamp: "2026-04-23T14:21:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T14:21:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T14:21:33Z" name: switch-graph-feffa-7cd874cdb5-n9vxb.18a902509814221d namespace: kserve-ci-e2e-test resourceVersion: "33282" uid: 33d0d95d-c52f-4a64-916e-8fcae4ca5c88 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-207.ec2.internal source: component: kubelet host: ip-10-0-132-207.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-feffa-7cd874cdb5 namespace: kserve-ci-e2e-test resourceVersion: "25814" uid: 4105d6b6-d110-40df-a157-620deaaf21a2 kind: Event lastTimestamp: "2026-04-23T14:04:50Z" message: 'Created pod: switch-graph-feffa-7cd874cdb5-n9vxb' metadata: creationTimestamp: "2026-04-23T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:50Z" name: switch-graph-feffa-7cd874cdb5.18a9016bca09b15a namespace: kserve-ci-e2e-test resourceVersion: "25818" uid: 4923f661-7da6-4894-b37b-fd97bdbeb04a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-feffa namespace: kserve-ci-e2e-test resourceVersion: "25812" uid: a5d191b3-55da-4530-a44b-ee164ddfd2ca kind: Event lastTimestamp: "2026-04-23T14:04:50Z" message: Scaled up replica set switch-graph-feffa-7cd874cdb5 from 0 to 1 metadata: creationTimestamp: "2026-04-23T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:04:50Z" name: switch-graph-feffa.18a9016bc96ff732 namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: a42d6eb0-1462-43cd-949a-8278f2974c23 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T14:04:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-feffa namespace: kserve-ci-e2e-test resourceVersion: "25831" uid: be851c37-fafa-4366-bba7-a730fe9e7fbe kind: Event lastTimestamp: "2026-04-23T14:04:50Z" message: InferenceGraph [switch-graph-feffa] is Ready metadata: creationTimestamp: "2026-04-23T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T14:04:50Z" name: switch-graph-feffa.18a9016bccf31061 namespace: kserve-ci-e2e-test resourceVersion: "25833" uid: c0fc05c9-6a53-4563-b5b9-64096121ecbd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T14:05:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-feffa namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: 6d39cd0d-86ef-40d8-a3db-da6909e9706f kind: Event lastTimestamp: "2026-04-23T14:05:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:35Z" name: switch-graph-feffa.18a9016f47f83533 namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 797a184b-7e47-4445-aae5-1e9116d62610 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T14:05:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-feffa namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: 6d39cd0d-86ef-40d8-a3db-da6909e9706f kind: Event lastTimestamp: "2026-04-23T14:05:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T14:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T14:05:35Z" name: switch-graph-feffa.18a9016f48681f22 namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: 6701cdb6-0cd5-4ea0-aa32-07cfac8fde28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33289"