--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:22:58.726546Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5d865-8ddbb586f-r2pd9 namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: a97bb6ee-8a64-4279-bb30-70f083352523 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-5d865-8ddbb586f-r2pd9 to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:22:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:22:58Z" name: ensemble-graph-5d865-8ddbb586f-r2pd9.18a90f81f006f4b1 namespace: kserve-ci-e2e-test resourceVersion: "22118" uid: dee523ba-c99e-4653-8005-3ba685dbff98 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:58Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5d865-8ddbb586f-r2pd9 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: a97bb6ee-8a64-4279-bb30-70f083352523 kind: Event lastTimestamp: "2026-04-23T18:22:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-5d865-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:58Z" name: ensemble-graph-5d865-8ddbb586f-r2pd9.18a90f81fad6b163 namespace: kserve-ci-e2e-test resourceVersion: "22130" uid: e1003855-f5d0-41e1-98b1-d132c39f61c5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:59Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5d865-8ddbb586f-r2pd9 namespace: kserve-ci-e2e-test resourceVersion: "22122" uid: a97bb6ee-8a64-4279-bb30-70f083352523 kind: Event lastTimestamp: "2026-04-23T18:22:59Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:22:59Z" name: ensemble-graph-5d865-8ddbb586f-r2pd9.18a90f822ca77c45 namespace: kserve-ci-e2e-test resourceVersion: "22142" uid: a01abb08-e880-4ecc-9f30-35410f591ba2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5d865} kind: Pod name: ensemble-graph-5d865-8ddbb586f-r2pd9 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: a97bb6ee-8a64-4279-bb30-70f083352523 kind: Event lastTimestamp: "2026-04-23T18:22:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:59Z" name: ensemble-graph-5d865-8ddbb586f-r2pd9.18a90f822dd3e181 namespace: kserve-ci-e2e-test resourceVersion: "22144" uid: 791ee0bc-154f-4f75-aa91-c1113d72cd45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5d865} kind: Pod name: ensemble-graph-5d865-8ddbb586f-r2pd9 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: a97bb6ee-8a64-4279-bb30-70f083352523 kind: Event lastTimestamp: "2026-04-23T18:22:59Z" message: 'Created container: ensemble-graph-5d865' metadata: creationTimestamp: "2026-04-23T18:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:59Z" name: ensemble-graph-5d865-8ddbb586f-r2pd9.18a90f8230852c04 namespace: kserve-ci-e2e-test resourceVersion: "22145" uid: 3e5b109a-4ccf-40f5-9b96-54ebef7d6800 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5d865} kind: Pod name: ensemble-graph-5d865-8ddbb586f-r2pd9 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: a97bb6ee-8a64-4279-bb30-70f083352523 kind: Event lastTimestamp: "2026-04-23T18:22:59Z" message: Started container ensemble-graph-5d865 metadata: creationTimestamp: "2026-04-23T18:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:59Z" name: ensemble-graph-5d865-8ddbb586f-r2pd9.18a90f8230eff25f namespace: kserve-ci-e2e-test resourceVersion: "22146" uid: 7f069128-ba35-49f3-8112-d333b61fa359 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5d865} kind: Pod name: ensemble-graph-5d865-8ddbb586f-r2pd9 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: a97bb6ee-8a64-4279-bb30-70f083352523 kind: Event lastTimestamp: "2026-04-23T18:23:08Z" message: Stopping container ensemble-graph-5d865 metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:08Z" name: ensemble-graph-5d865-8ddbb586f-r2pd9.18a90f8448829f13 namespace: kserve-ci-e2e-test resourceVersion: "22239" uid: dff65c0f-40de-4297-8737-282e0bdaba21 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:23:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5d865} kind: Pod name: ensemble-graph-5d865-8ddbb586f-r2pd9 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: a97bb6ee-8a64-4279-bb30-70f083352523 kind: Event lastTimestamp: "2026-04-23T18:23:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:36Z" name: ensemble-graph-5d865-8ddbb586f-r2pd9.18a90f84ecd03d4b namespace: kserve-ci-e2e-test resourceVersion: "22592" uid: 6d4e8152-20df-4363-8072-a5d0e589c1c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-5d865-8ddbb586f namespace: kserve-ci-e2e-test resourceVersion: "22109" uid: dcc2cf0e-4fe8-435e-9a09-7bb053557c02 kind: Event lastTimestamp: "2026-04-23T18:22:58Z" message: 'Created pod: ensemble-graph-5d865-8ddbb586f-r2pd9' metadata: creationTimestamp: "2026-04-23T18:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:58Z" name: ensemble-graph-5d865-8ddbb586f.18a90f81ef9ecbaa namespace: kserve-ci-e2e-test resourceVersion: "22114" uid: 3734e2e9-a355-4e13-8665-e6ddf0c7dffe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-5d865 namespace: kserve-ci-e2e-test resourceVersion: "22106" uid: bd8dffbf-878c-4057-b507-7194c646b5b3 kind: Event lastTimestamp: "2026-04-23T18:22:58Z" message: Scaled up replica set ensemble-graph-5d865-8ddbb586f from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:58Z" name: ensemble-graph-5d865.18a90f81eefe9868 namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: 9a491bea-0a26-4334-b669-7fc339999f9d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:58Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5d865 namespace: kserve-ci-e2e-test resourceVersion: "22124" uid: e6cc201b-c570-479d-84e5-2f14412be124 kind: Event lastTimestamp: "2026-04-23T18:22:58Z" message: InferenceGraph [ensemble-graph-5d865] is Ready metadata: creationTimestamp: "2026-04-23T18:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:58Z" name: ensemble-graph-5d865.18a90f81f2fad82b namespace: kserve-ci-e2e-test resourceVersion: "22126" uid: f090880b-0396-407e-9ed5-0f58a423e9a7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:24:09.019707Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-83978-d4f856f6f-x2xqt namespace: kserve-ci-e2e-test resourceVersion: "23072" uid: 868c11ce-d2e3-4592-9d95-b9ac3f7ee8fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-83978-d4f856f6f-x2xqt to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:24:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:24:09Z" name: ensemble-graph-83978-d4f856f6f-x2xqt.18a90f924dd36b09 namespace: kserve-ci-e2e-test resourceVersion: "23076" uid: 7f2b10bb-fe22-4d0e-a6e6-37aaff44b2b2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:09Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-83978-d4f856f6f-x2xqt namespace: kserve-ci-e2e-test resourceVersion: "23074" uid: 868c11ce-d2e3-4592-9d95-b9ac3f7ee8fe kind: Event lastTimestamp: "2026-04-23T18:24:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-83978-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:09Z" name: ensemble-graph-83978-d4f856f6f-x2xqt.18a90f925924f419 namespace: kserve-ci-e2e-test resourceVersion: "23096" uid: 846926db-d71a-4ef2-9b97-5dde15740ca7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:10Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-83978-d4f856f6f-x2xqt namespace: kserve-ci-e2e-test resourceVersion: "23081" uid: 868c11ce-d2e3-4592-9d95-b9ac3f7ee8fe kind: Event lastTimestamp: "2026-04-23T18:24:10Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:24:10Z" name: ensemble-graph-83978-d4f856f6f-x2xqt.18a90f928aa21a00 namespace: kserve-ci-e2e-test resourceVersion: "23105" uid: 67e4780a-3861-43c0-97d3-a16807b4f2a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-83978} kind: Pod name: ensemble-graph-83978-d4f856f6f-x2xqt namespace: kserve-ci-e2e-test resourceVersion: "23074" uid: 868c11ce-d2e3-4592-9d95-b9ac3f7ee8fe kind: Event lastTimestamp: "2026-04-23T18:24:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:10Z" name: ensemble-graph-83978-d4f856f6f-x2xqt.18a90f928bcb803a namespace: kserve-ci-e2e-test resourceVersion: "23107" uid: 01411ef2-e4d3-45e3-9237-666587ba4d0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-83978} kind: Pod name: ensemble-graph-83978-d4f856f6f-x2xqt namespace: kserve-ci-e2e-test resourceVersion: "23074" uid: 868c11ce-d2e3-4592-9d95-b9ac3f7ee8fe kind: Event lastTimestamp: "2026-04-23T18:24:10Z" message: 'Created container: ensemble-graph-83978' metadata: creationTimestamp: "2026-04-23T18:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:10Z" name: ensemble-graph-83978-d4f856f6f-x2xqt.18a90f928eb93ed3 namespace: kserve-ci-e2e-test resourceVersion: "23108" uid: 120f6497-3abb-46eb-b3d0-4c2205183ca9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-83978} kind: Pod name: ensemble-graph-83978-d4f856f6f-x2xqt namespace: kserve-ci-e2e-test resourceVersion: "23074" uid: 868c11ce-d2e3-4592-9d95-b9ac3f7ee8fe kind: Event lastTimestamp: "2026-04-23T18:24:10Z" message: Started container ensemble-graph-83978 metadata: creationTimestamp: "2026-04-23T18:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:10Z" name: ensemble-graph-83978-d4f856f6f-x2xqt.18a90f928f2d8f46 namespace: kserve-ci-e2e-test resourceVersion: "23109" uid: e4f8d132-ffda-4cf8-a731-b20e174d04a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-83978} kind: Pod name: ensemble-graph-83978-d4f856f6f-x2xqt namespace: kserve-ci-e2e-test resourceVersion: "23074" uid: 868c11ce-d2e3-4592-9d95-b9ac3f7ee8fe kind: Event lastTimestamp: "2026-04-23T18:32:23Z" message: Stopping container ensemble-graph-83978 metadata: creationTimestamp: "2026-04-23T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:23Z" name: ensemble-graph-83978-d4f856f6f-x2xqt.18a9100580404721 namespace: kserve-ci-e2e-test resourceVersion: "26953" uid: bd4389f8-1387-48d3-877c-c6af0b6a10f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:32:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-83978} kind: Pod name: ensemble-graph-83978-d4f856f6f-x2xqt namespace: kserve-ci-e2e-test resourceVersion: "23074" uid: 868c11ce-d2e3-4592-9d95-b9ac3f7ee8fe kind: Event lastTimestamp: "2026-04-23T18:32:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:51Z" name: ensemble-graph-83978-d4f856f6f-x2xqt.18a91006329b8659 namespace: kserve-ci-e2e-test resourceVersion: "27295" uid: e854f648-c1b3-4a23-87b6-f0b2cb393e5f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-83978-d4f856f6f namespace: kserve-ci-e2e-test resourceVersion: "23068" uid: 8ad6604b-288f-4abd-bd42-c613fc5abb1c kind: Event lastTimestamp: "2026-04-23T18:24:09Z" message: 'Created pod: ensemble-graph-83978-d4f856f6f-x2xqt' metadata: creationTimestamp: "2026-04-23T18:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:09Z" name: ensemble-graph-83978-d4f856f6f.18a90f924d4f9883 namespace: kserve-ci-e2e-test resourceVersion: "23073" uid: 95f0d6be-29fe-4051-990e-acedec4095da reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-83978 namespace: kserve-ci-e2e-test resourceVersion: "23065" uid: f6a96c6c-9724-4a18-8ddd-26dbe333dac0 kind: Event lastTimestamp: "2026-04-23T18:24:08Z" message: Scaled up replica set ensemble-graph-83978-d4f856f6f from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:24:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:08Z" name: ensemble-graph-83978.18a90f924c8bf378 namespace: kserve-ci-e2e-test resourceVersion: "23070" uid: 68078db1-f899-46da-990e-e2a6357ee63a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-83978 namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: 624f546c-ee4a-4b0c-8fef-c4fae93816a3 kind: Event lastTimestamp: "2026-04-23T18:24:09Z" message: InferenceGraph [ensemble-graph-83978] is Ready metadata: creationTimestamp: "2026-04-23T18:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:09Z" name: ensemble-graph-83978.18a90f9251c3f210 namespace: kserve-ci-e2e-test resourceVersion: "23088" uid: 9da49706-3901-456a-a643-5d5e1fa1e1fc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-83978 namespace: kserve-ci-e2e-test resourceVersion: "23058" uid: 624f546c-ee4a-4b0c-8fef-c4fae93816a3 kind: Event lastTimestamp: "2026-04-23T18:24:09Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-83978": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-83978": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:09Z" name: ensemble-graph-83978.18a90f9254421879 namespace: kserve-ci-e2e-test resourceVersion: "23093" uid: 647d4cf1-28e2-465c-8d99-9f6c1bc86df3 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-83978 namespace: kserve-ci-e2e-test resourceVersion: "23058" uid: 624f546c-ee4a-4b0c-8fef-c4fae93816a3 kind: Event lastTimestamp: "2026-04-23T18:24:09Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-83978": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:09Z" name: ensemble-graph-83978.18a90f9254426034 namespace: kserve-ci-e2e-test resourceVersion: "23094" uid: d8e3c734-aa90-44fc-a368-1e88e3c707a2 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:24:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-83978 namespace: kserve-ci-e2e-test resourceVersion: "23067" uid: f9531629-ccd2-417a-9b7a-0d5018181a26 kind: Event lastTimestamp: "2026-04-23T18:24:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:39Z" name: ensemble-graph-83978.18a90f95cae211a6 namespace: kserve-ci-e2e-test resourceVersion: "23510" uid: 56259e1f-1fde-4bc6-9cb2-a94d38d7ed1f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:24:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-83978 namespace: kserve-ci-e2e-test resourceVersion: "23067" uid: f9531629-ccd2-417a-9b7a-0d5018181a26 kind: Event lastTimestamp: "2026-04-23T18:24:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:39Z" name: ensemble-graph-83978.18a90f95cb4cb882 namespace: kserve-ci-e2e-test resourceVersion: "23515" uid: cd78db58-8de9-483e-8e89-cf452bf249c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:24:04.868800Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22972" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-02052-predictor-7ff4bf9679-cdttd to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:24:04Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a90f915669a299 namespace: kserve-ci-e2e-test resourceVersion: "22976" uid: 6669bf5b-eafe-4af4-9bfd-517e10d86139 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22980" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:24:05Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a90f917334534d namespace: kserve-ci-e2e-test resourceVersion: "22995" uid: f9dd3fed-b192-470b-a785-05f4376937c7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a90f91759fd2bf namespace: kserve-ci-e2e-test resourceVersion: "22999" uid: d08ab10b-c4ec-4581-b810-6ea3efbd789f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a90f91788a7401 namespace: kserve-ci-e2e-test resourceVersion: "23002" uid: 45aa7cf2-b3a6-41a9-8812-6e81de0446c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a90f91790364eb namespace: kserve-ci-e2e-test resourceVersion: "23003" uid: 994b11f3-e5c7-4b25-b98f-2edcd4542c0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a90f917910ffb0 namespace: kserve-ci-e2e-test resourceVersion: "23004" uid: c021beba-fbc5-453a-8571-b690de9630ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a90f917c66d4ad namespace: kserve-ci-e2e-test resourceVersion: "23006" uid: 8ade7c6b-e498-47b9-bd51-6906a8b973bd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a90f917ce13533 namespace: kserve-ci-e2e-test resourceVersion: "23007" uid: 1d6889e3-8e7d-4988-98b4-1a14a9afa913 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:24:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:33:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:24:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:33Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a90f91da614780 namespace: kserve-ci-e2e-test resourceVersion: "27810" uid: c5764f35-409e-481c-bd70-dbe94ccc8d76 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:29Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a91014d3b335d0 namespace: kserve-ci-e2e-test resourceVersion: "27680" uid: 34497427-8d63-4517-ae23-0629ea12d24e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:29Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a91014d3b46289 namespace: kserve-ci-e2e-test resourceVersion: "27685" uid: 8d4fa59e-18c9-4126-a84b-159993aefe22 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a04b6319-9573-4718-96c5-c45ad07a9a8f kind: Event lastTimestamp: "2026-04-23T18:33:33Z" message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": dial tcp 10.133.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:33Z" name: error-404-isvc-02052-predictor-7ff4bf9679-cdttd.18a91015a2fc0088 namespace: kserve-ci-e2e-test resourceVersion: "27809" uid: 1e0663bd-f2a3-480e-8446-a8ce2e051ff0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-02052-predictor-7ff4bf9679 namespace: kserve-ci-e2e-test resourceVersion: "22965" uid: c645c996-fba6-4f2d-bfa9-6fa789cfbbb9 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: 'Created pod: error-404-isvc-02052-predictor-7ff4bf9679-cdttd' metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:04Z" name: error-404-isvc-02052-predictor-7ff4bf9679.18a90f9156009321 namespace: kserve-ci-e2e-test resourceVersion: "22973" uid: 6e73c2de-ea1f-4816-b434-4a4542a7d5d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "22962" uid: 04ccdae0-7399-44cf-8c15-23eb006bfc3a kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: Scaled up replica set error-404-isvc-02052-predictor-7ff4bf9679 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:04Z" name: error-404-isvc-02052-predictor.18a90f9154647946 namespace: kserve-ci-e2e-test resourceVersion: "22967" uid: 42545ebb-d8fe-4124-bdb1-561c11098c2d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:24:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "22964" uid: 9b1497f7-ac85-47e4-a049-76b110757aa7 kind: Event lastTimestamp: "2026-04-23T18:24:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:34Z" name: error-404-isvc-02052-predictor.18a90f94d2d34da3 namespace: kserve-ci-e2e-test resourceVersion: "23319" uid: cfc59139-8c9e-4d07-a13a-0cc603aef77d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:24:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "22964" uid: 9b1497f7-ac85-47e4-a049-76b110757aa7 kind: Event lastTimestamp: "2026-04-23T18:24:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:34Z" name: error-404-isvc-02052-predictor.18a90f94d32125d4 namespace: kserve-ci-e2e-test resourceVersion: "23320" uid: 169600b9-97ec-41e2-ae0c-c4362940a398 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:24:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "23194" uid: 9b1497f7-ac85-47e4-a049-76b110757aa7 kind: Event lastTimestamp: "2026-04-23T18:25:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:34Z" name: error-404-isvc-02052-predictor.18a90f9bd091bcb1 namespace: kserve-ci-e2e-test resourceVersion: "24123" uid: e4d36ca9-c98d-4c98-999c-d926910f301d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:24:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "23194" uid: 9b1497f7-ac85-47e4-a049-76b110757aa7 kind: Event lastTimestamp: "2026-04-23T18:25:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:34Z" name: error-404-isvc-02052-predictor.18a90f9bd105f382 namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: b6198798-825c-4913-99d5-3daaa152128c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02052 namespace: kserve-ci-e2e-test resourceVersion: "22930" uid: c11c333c-46a9-41f9-b7f1-c2658d47a329 kind: Event lastTimestamp: "2026-04-23T18:24:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:53Z" name: error-404-isvc-02052.18a90f91543accc5 namespace: kserve-ci-e2e-test resourceVersion: "23758" uid: 768d9ab2-a5e4-49da-a909-b1bce283e23a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02052 namespace: kserve-ci-e2e-test resourceVersion: "22930" uid: c11c333c-46a9-41f9-b7f1-c2658d47a329 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-02052": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-02052": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:04Z" name: error-404-isvc-02052.18a90f915486a4c6 namespace: kserve-ci-e2e-test resourceVersion: "22969" uid: 8405a448-caeb-4502-b26f-2e475a333ed5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02052 namespace: kserve-ci-e2e-test resourceVersion: "22930" uid: c11c333c-46a9-41f9-b7f1-c2658d47a329 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-02052": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:04Z" name: error-404-isvc-02052.18a90f915486fcab namespace: kserve-ci-e2e-test resourceVersion: "22970" uid: 1d079089-f81d-43ab-9dca-8f73fe8c0730 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02052 namespace: kserve-ci-e2e-test resourceVersion: "23756" uid: c11c333c-46a9-41f9-b7f1-c2658d47a329 kind: Event lastTimestamp: "2026-04-23T18:24:53Z" message: InferenceService [error-404-isvc-02052] is Ready metadata: creationTimestamp: "2026-04-23T18:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:53Z" name: error-404-isvc-02052.18a90f9c951d48ef namespace: kserve-ci-e2e-test resourceVersion: "23757" uid: 961f0b03-5352-4408-81df-600e9124cf40 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:12:33.782875Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-03167-predictor-659dbf79ff-tgwf9 to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:12:33Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef06e7b4567 namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: fca383f6-57f2-44f1-8dd6-342110a8811d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:12:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-03167-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:34Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef0798a8716 namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: c84f2954-c67c-4efc-a26f-238ea80003f1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:12:35Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:12:35Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef0e1e600e4 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 88252dbb-b9a4-4b03-a08f-e1efe7835b76 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:12:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:35Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef0e34592b9 namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: d021b68e-475a-49d4-8a89-5e4723f611ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:12:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:35Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef0e685d857 namespace: kserve-ci-e2e-test resourceVersion: "17004" uid: fb53eaa0-04d8-473a-98ef-a61e0e970a25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:12:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:35Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef0e703a834 namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: 9891e5b8-5977-40aa-ba49-1d1499b0dcc0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:12:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:35Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef0e7111a86 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: 7c51819c-2d8a-4373-ad1b-8be541ebfa0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:12:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:35Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef0eb58ea3e namespace: kserve-ci-e2e-test resourceVersion: "17008" uid: f0221162-b36f-4096-adf4-c2c322411975 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:12:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:35Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef0ebfc3f41 namespace: kserve-ci-e2e-test resourceVersion: "17009" uid: 3b58a93e-379e-4d40-88bc-9cd7729cf07f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:13:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:13Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90ef157c5e558 namespace: kserve-ci-e2e-test resourceVersion: "17384" uid: 2ed4fbe3-bcd0-4acf-b5d9-4afb66240d2f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:58Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90f73f3810315 namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 78726170-74f0-411e-a7da-439af3fa38fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:58Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90f73f3818834 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: ad80b1d5-d4b2-4993-9773-21ba508ef493 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9 namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: cd83c524-2cee-4a78-b741-74d2bed83cce kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:58Z" name: error-404-isvc-03167-predictor-659dbf79ff-tgwf9.18a90f73f635caec namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 5416be8e-4908-44aa-91a0-c2f8568d7b8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-03167-predictor-659dbf79ff namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: 61bab879-50b9-4d29-812e-42ea12d6055c kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: 'Created pod: error-404-isvc-03167-predictor-659dbf79ff-tgwf9' metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:33Z" name: error-404-isvc-03167-predictor-659dbf79ff.18a90ef06dede2af namespace: kserve-ci-e2e-test resourceVersion: "16955" uid: dc513982-d18a-4619-a51e-082a04d5bc8e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: a6298287-983f-40bc-adf5-fb25e4c19217 kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: Scaled up replica set error-404-isvc-03167-predictor-659dbf79ff from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:33Z" name: error-404-isvc-03167-predictor.18a90ef06be42c5e namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: f18b143f-991c-453c-9f4a-393bb82655ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:12:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: c348f6a1-370f-422d-a77f-b6556bd7350e kind: Event lastTimestamp: "2026-04-23T18:13:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:18Z" name: error-404-isvc-03167-predictor.18a90ef3ea944eed namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: c06ed423-ef7d-4c8d-a9a6-56cfbeb331eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:12:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: c348f6a1-370f-422d-a77f-b6556bd7350e kind: Event lastTimestamp: "2026-04-23T18:13:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:18Z" name: error-404-isvc-03167-predictor.18a90ef3eb0cbdc7 namespace: kserve-ci-e2e-test resourceVersion: "17438" uid: 46512475-a0ab-47f5-b612-3d349ad46a09 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "17156" uid: c348f6a1-370f-422d-a77f-b6556bd7350e kind: Event lastTimestamp: "2026-04-23T18:13:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:48Z" name: error-404-isvc-03167-predictor.18a90efe675d56db namespace: kserve-ci-e2e-test resourceVersion: "17899" uid: 43a719e2-3dea-4aec-9f80-348f23da892c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "17156" uid: c348f6a1-370f-422d-a77f-b6556bd7350e kind: Event lastTimestamp: "2026-04-23T18:13:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:48Z" name: error-404-isvc-03167-predictor.18a90efe67cc23ee namespace: kserve-ci-e2e-test resourceVersion: "17900" uid: 0f523e8b-7bd4-4ab4-83e2-bbd3960102b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03167 namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 2090664b-33e5-4514-8ab0-e2929605c1aa kind: Event lastTimestamp: "2026-04-23T18:13:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:23Z" name: error-404-isvc-03167.18a90ef06be54ac5 namespace: kserve-ci-e2e-test resourceVersion: "17646" uid: 48ffa25d-2a70-4460-ba64-b206f080797a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03167 namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 2090664b-33e5-4514-8ab0-e2929605c1aa kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-03167": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-03167": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:33Z" name: error-404-isvc-03167.18a90ef06c67bf43 namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: 687feac9-0365-4986-b1f4-f076a47be650 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03167 namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 2090664b-33e5-4514-8ab0-e2929605c1aa kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-03167": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:33Z" name: error-404-isvc-03167.18a90ef06c680540 namespace: kserve-ci-e2e-test resourceVersion: "16952" uid: d14683ce-8db1-4658-bcda-41cac9e30c1c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03167 namespace: kserve-ci-e2e-test resourceVersion: "17644" uid: 2090664b-33e5-4514-8ab0-e2929605c1aa kind: Event lastTimestamp: "2026-04-23T18:13:23Z" message: InferenceService [error-404-isvc-03167] is Ready metadata: creationTimestamp: "2026-04-23T18:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:23Z" name: error-404-isvc-03167.18a90efc12ce6772 namespace: kserve-ci-e2e-test resourceVersion: "17645" uid: 3abca141-5d3d-4ac5-a5e5-baeb554bc94f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:21:58.775313Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5d865-predictor-5bf4578b77-phhwj to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:21:58Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f73faa7ada1 namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 8afe1ebe-5f19-4e00-af74-9d61726d6773 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:21:59Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f7413fed0d7 namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 69f558b6-7a85-4f7b-bad3-48c6eedae701 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f74154e4a2a namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 609c8f8e-745a-4c2e-b8b7-b2155c8b2f6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f741820e232 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: ae7f3097-814b-4f91-a06c-8b312172b669 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f7418866fee namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: f9accc7a-34f1-4439-a7e4-102f555b4684 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f741892373a namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: 96ab2f3f-14ba-4947-8778-6175a89135b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f741b61e1ed namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: 94df1561-965d-41bc-aea9-08a8ef4b4fd1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f741bc7bb66 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: d82e0fc9-5e7a-417b-a188-86e96799029e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:22:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:35Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f742bc520d4 namespace: kserve-ci-e2e-test resourceVersion: "21755" uid: 5ef6bd06-b858-41c3-ac5c-114d9ba427df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:23:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:08Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f844c3d8885 namespace: kserve-ci-e2e-test resourceVersion: "22257" uid: c7d7752d-59bf-4e33-b276-4f37e12cf18f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:23:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:08Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f844c407836 namespace: kserve-ci-e2e-test resourceVersion: "22259" uid: c9f6c354-cc05-4890-afd5-a6a26d2ced5c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 93f60042-dc14-4c57-a360-01d9ac01b871 kind: Event lastTimestamp: "2026-04-23T18:23:10Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:10Z" name: error-404-isvc-5d865-predictor-5bf4578b77-phhwj.18a90f84b4043a86 namespace: kserve-ci-e2e-test resourceVersion: "22367" uid: e6ab2e7c-5f64-4e15-8bcf-6c6d7ba78233 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5d865-predictor-5bf4578b77 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: 5fdb4771-f77f-4741-9a88-019268706566 kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: 'Created pod: error-404-isvc-5d865-predictor-5bf4578b77-phhwj' metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:21:58Z" name: error-404-isvc-5d865-predictor-5bf4578b77.18a90f73fa3948da namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 7edc3be5-849c-4aa8-b826-5d3725ace024 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: edc73738-7fbc-4333-996d-b463efec45ee kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: Scaled up replica set error-404-isvc-5d865-predictor-5bf4578b77 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:21:58Z" name: error-404-isvc-5d865-predictor.18a90f73f858349a namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: 2fa168dc-74b5-49ec-bfea-5b5f70cfefcb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: 64d1c9e6-81ee-420e-ad62-f7579cd96f62 kind: Event lastTimestamp: "2026-04-23T18:22:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:28Z" name: error-404-isvc-5d865-predictor.18a90f7776da53a6 namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 07c5b5e1-81c7-413e-94ed-e59782fc976a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: 64d1c9e6-81ee-420e-ad62-f7579cd96f62 kind: Event lastTimestamp: "2026-04-23T18:22:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:28Z" name: error-404-isvc-5d865-predictor.18a90f77774c0589 namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: 11cad6ff-44b5-4547-b292-11a23c3fdb23 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: 64d1c9e6-81ee-420e-ad62-f7579cd96f62 kind: Event lastTimestamp: "2026-04-23T18:22:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:22:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:58Z" name: error-404-isvc-5d865-predictor.18a90f7e74be9ec0 namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: ffa58cb4-7ce1-43a1-bbc2-19d1981a7d75 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: 64d1c9e6-81ee-420e-ad62-f7579cd96f62 kind: Event lastTimestamp: "2026-04-23T18:22:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:22:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:58Z" name: error-404-isvc-5d865-predictor.18a90f7e7529295e namespace: kserve-ci-e2e-test resourceVersion: "22128" uid: 791e94f2-5e89-4384-89ec-1af86812593e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 4258d580-e8b5-4991-9dc7-f3ffeed16847 kind: Event lastTimestamp: "2026-04-23T18:22:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:46Z" name: error-404-isvc-5d865.18a90f73f8225259 namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: b39c1cb6-7fbb-4112-9a1a-edcd7c19c09d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 4258d580-e8b5-4991-9dc7-f3ffeed16847 kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: 'Failed to update status for InferenceService "error-404-isvc-5d865": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5d865": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:21:58Z" name: error-404-isvc-5d865.18a90f73f87241dd namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: c6b5b26a-4398-46b3-b9f7-b6b9a424e9e6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 4258d580-e8b5-4991-9dc7-f3ffeed16847 kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5d865": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:21:58Z" name: error-404-isvc-5d865.18a90f73f87286d5 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 0fc60fac-9f90-4226-959d-b74127a5e6e1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 4258d580-e8b5-4991-9dc7-f3ffeed16847 kind: Event lastTimestamp: "2026-04-23T18:22:46Z" message: InferenceService [error-404-isvc-5d865] is Ready metadata: creationTimestamp: "2026-04-23T18:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:46Z" name: error-404-isvc-5d865.18a90f7f0e338020 namespace: kserve-ci-e2e-test resourceVersion: "21979" uid: 32e8be35-699d-403d-9a92-11999291b6e8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:32:24.118753Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-60f5a-predictor-7646d945df-9jfr7 to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91005940755d0 namespace: kserve-ci-e2e-test resourceVersion: "27040" uid: 825365a5-2a47-4e74-9488-723d6e59cf41 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27043" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91005ad5e1043 namespace: kserve-ci-e2e-test resourceVersion: "27052" uid: 3960f68c-7ece-4142-a673-4a211be4fa3c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91005aec63eb7 namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: c77ba182-6a7d-4595-95c1-1bdd5cd26472 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91005b198f7af namespace: kserve-ci-e2e-test resourceVersion: "27055" uid: 2cfcf6a6-2710-46ed-adc4-8a2af206b76f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91005b20bdb5b namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: 1e5e69a7-71be-421c-8700-01d520bbc0bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91005b2175047 namespace: kserve-ci-e2e-test resourceVersion: "27057" uid: de58991d-b304-4841-9115-9e3ca47eceab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91005b542f948 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: 3c310fe1-582c-4313-8a20-b817e0ee6127 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91005b5c7c847 namespace: kserve-ci-e2e-test resourceVersion: "27062" uid: e9889d68-1507-4815-9e50-6576f20d766a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:33:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:41Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91005f90e38db namespace: kserve-ci-e2e-test resourceVersion: "28018" uid: 198442ab-eeb7-4783-828c-e8d943a5a6d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91016dcf01b12 namespace: kserve-ci-e2e-test resourceVersion: "27943" uid: 4a5ee753-45a3-423d-88dc-93f672487f5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91016dcf0986e namespace: kserve-ci-e2e-test resourceVersion: "27947" uid: 00f19199-8362-457e-bc98-27af4b45a3a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: bff60ad1-536a-4cf2-832c-a469d7aba58b kind: Event lastTimestamp: "2026-04-23T18:33:41Z" message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp 10.133.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:41Z" name: error-404-isvc-60f5a-predictor-7646d945df-9jfr7.18a91017ab374944 namespace: kserve-ci-e2e-test resourceVersion: "28015" uid: 3c03a281-91dd-4335-967e-e6e050f7cef0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-60f5a-predictor-7646d945df namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: 95f5cc99-4d02-458b-ab35-90037f42f190 kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: 'Created pod: error-404-isvc-60f5a-predictor-7646d945df-9jfr7' metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor-7646d945df.18a910059328a979 namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: 3080cf4c-3d81-4787-871b-ee8edaf3c822 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 4218e9b7-f176-4ea2-b54c-121769854371 kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: Scaled up replica set error-404-isvc-60f5a-predictor-7646d945df from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a-predictor.18a91005914be173 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: e355f2f7-a410-4ffc-8150-2fccec8a684d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:32:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: c783358d-963e-4842-b5b4-60cb67dc457d kind: Event lastTimestamp: "2026-04-23T18:32:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:32:54Z" name: error-404-isvc-60f5a-predictor.18a910090f935135 namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: 0362105b-9788-4173-bad8-1560a82c88da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:32:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: c783358d-963e-4842-b5b4-60cb67dc457d kind: Event lastTimestamp: "2026-04-23T18:32:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:32:54Z" name: error-404-isvc-60f5a-predictor.18a910090ff93001 namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 36195c65-8998-4818-bb79-9bf57774ddd8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:33:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27203" uid: c783358d-963e-4842-b5b4-60cb67dc457d kind: Event lastTimestamp: "2026-04-23T18:33:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:33:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:24Z" name: error-404-isvc-60f5a-predictor.18a910100d4b3cec namespace: kserve-ci-e2e-test resourceVersion: "27582" uid: b4344c5f-8e9f-488c-977f-a5b843d10a11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:33:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27203" uid: c783358d-963e-4842-b5b4-60cb67dc457d kind: Event lastTimestamp: "2026-04-23T18:33:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:33:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:24Z" name: error-404-isvc-60f5a-predictor.18a910100db26e34 namespace: kserve-ci-e2e-test resourceVersion: "27583" uid: 48fb0e5e-debb-45c6-b0ef-6ca0576f639b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60f5a namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: 336d1cf8-0617-44a5-85af-1b65d00c1dc6 kind: Event lastTimestamp: "2026-04-23T18:33:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:12Z" name: error-404-isvc-60f5a.18a9100590f4278d namespace: kserve-ci-e2e-test resourceVersion: "27467" uid: ec0aa2cd-51d9-4426-8b24-891f99dba3c5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60f5a namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: 336d1cf8-0617-44a5-85af-1b65d00c1dc6 kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: 'Failed to update status for InferenceService "error-404-isvc-60f5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-60f5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a.18a91005919873eb namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 3b816810-a516-4354-b20e-7c658f6872b7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60f5a namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: 336d1cf8-0617-44a5-85af-1b65d00c1dc6 kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-60f5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:32:24Z" name: error-404-isvc-60f5a.18a910059198bd56 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 1575c11c-4d63-4de0-bf6b-b4038e6b59f2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60f5a namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: 336d1cf8-0617-44a5-85af-1b65d00c1dc6 kind: Event lastTimestamp: "2026-04-23T18:33:11Z" message: InferenceService [error-404-isvc-60f5a] is Ready metadata: creationTimestamp: "2026-04-23T18:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:11Z" name: error-404-isvc-60f5a.18a91010b6628096 namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: b08072e1-221c-43f5-8508-e63eeb685c5d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:33:38.404741Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27950" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a91016dfd19752 namespace: kserve-ci-e2e-test resourceVersion: "27955" uid: 2264699c-3a0f-4ede-908b-7b47938c1e54 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27960" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a91016f9b59144 namespace: kserve-ci-e2e-test resourceVersion: "27974" uid: b193c6b8-f6f6-4fb9-8148-639a43dd6d04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a91016fb248e0f namespace: kserve-ci-e2e-test resourceVersion: "27976" uid: c655fc8c-0fbf-46f4-b7ef-03e6b01c8d10 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a91016fe78f61e namespace: kserve-ci-e2e-test resourceVersion: "27979" uid: 1f007e0d-9a6c-4247-9ba8-c21df0557fa2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a91016feea69c0 namespace: kserve-ci-e2e-test resourceVersion: "27980" uid: b2bd9901-87de-4b87-b0d8-c3e0223804a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a91016fef6c0d4 namespace: kserve-ci-e2e-test resourceVersion: "27981" uid: ba08e4ed-c187-4dcd-b74e-bc1275decb94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a9101702d2864f namespace: kserve-ci-e2e-test resourceVersion: "27982" uid: 172b828b-76a0-4d10-90c6-cbd0896de62d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:39Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a91017034b6e33 namespace: kserve-ci-e2e-test resourceVersion: "27983" uid: 699bd6e7-94da-4082-b69b-3dd11dc4126a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:33:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:42:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:33:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:42:56Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a9101745738c93 namespace: kserve-ci-e2e-test resourceVersion: "32066" uid: 88eb9619-1bc0-482c-b0aa-28424da3f726 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:42:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:42:53Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a910980d93864c namespace: kserve-ci-e2e-test resourceVersion: "32039" uid: fb5c404c-1f40-4b68-92d2-b419319be507 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:42:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:42:53Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a910980d94086a namespace: kserve-ci-e2e-test resourceVersion: "32040" uid: 5a07d727-768e-42e6-9de9-206f0eb27c7e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:42:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh namespace: kserve-ci-e2e-test resourceVersion: "27953" uid: c8dc829a-d241-4993-b315-4bb04d2cff26 kind: Event lastTimestamp: "2026-04-23T18:42:56Z" message: 'Readiness probe failed: Get "https://10.133.0.47:8643/healthz": dial tcp 10.133.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:42:56Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh.18a91098b9c95c8c namespace: kserve-ci-e2e-test resourceVersion: "32063" uid: d5d35037-11a1-4a41-86eb-62ce73c66a1c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-760ed-predictor-5fd5f5bd9 namespace: kserve-ci-e2e-test resourceVersion: "27940" uid: 86c8015a-5318-4df0-9d80-c0b946fd8ce8 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'Created pod: error-404-isvc-760ed-predictor-5fd5f5bd9-f8hbh' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed-predictor-5fd5f5bd9.18a91016deeafc3e namespace: kserve-ci-e2e-test resourceVersion: "27951" uid: 26b7f13f-d516-45cd-b7d5-0481f33043e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "27934" uid: 7a002e52-8224-42f0-bb1e-7f6800bb14ba kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Scaled up replica set error-404-isvc-760ed-predictor-5fd5f5bd9 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed-predictor.18a91016dd1327b9 namespace: kserve-ci-e2e-test resourceVersion: "27944" uid: 2401e050-d4bd-47b0-8ee8-76aa03c30ef4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:33:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "27938" uid: a288eb1a-01df-429e-9ced-2cc3f4138525 kind: Event lastTimestamp: "2026-04-23T18:34:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:08Z" name: error-404-isvc-760ed-predictor.18a9101a5b0794c6 namespace: kserve-ci-e2e-test resourceVersion: "28254" uid: 87d4f984-cf7c-46cc-baa2-8f3468be4cfa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:33:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "27938" uid: a288eb1a-01df-429e-9ced-2cc3f4138525 kind: Event lastTimestamp: "2026-04-23T18:34:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:08Z" name: error-404-isvc-760ed-predictor.18a9101a5b74ef2b namespace: kserve-ci-e2e-test resourceVersion: "28255" uid: 0667e01e-243f-43b7-8550-223a8039654e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:34:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "28128" uid: a288eb1a-01df-429e-9ced-2cc3f4138525 kind: Event lastTimestamp: "2026-04-23T18:35:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:35:08Z" name: error-404-isvc-760ed-predictor.18a9102158f42c13 namespace: kserve-ci-e2e-test resourceVersion: "28830" uid: 2f054f8d-7a49-4cf3-9edc-8d60951cf96d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:34:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "28128" uid: a288eb1a-01df-429e-9ced-2cc3f4138525 kind: Event lastTimestamp: "2026-04-23T18:35:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:35:08Z" name: error-404-isvc-760ed-predictor.18a91021596a2fd6 namespace: kserve-ci-e2e-test resourceVersion: "28831" uid: cb23e235-98c5-43f8-9e89-a86e7e648eb0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-760ed namespace: kserve-ci-e2e-test resourceVersion: "27910" uid: b208670c-5c90-46ad-86fb-86d6a92a8f5d kind: Event lastTimestamp: "2026-04-23T18:34:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:36Z" name: error-404-isvc-760ed.18a91016dc5c8e15 namespace: kserve-ci-e2e-test resourceVersion: "28506" uid: de95bdf8-0010-4ddb-895d-6d4faa5b6158 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-760ed namespace: kserve-ci-e2e-test resourceVersion: "27910" uid: b208670c-5c90-46ad-86fb-86d6a92a8f5d kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'Failed to update status for InferenceService "error-404-isvc-760ed": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-760ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed.18a91016dd1a55d0 namespace: kserve-ci-e2e-test resourceVersion: "27945" uid: f93d5d34-ceda-4021-a23a-58250120cc8e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-760ed namespace: kserve-ci-e2e-test resourceVersion: "27910" uid: b208670c-5c90-46ad-86fb-86d6a92a8f5d kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-760ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:38Z" name: error-404-isvc-760ed.18a91016dd1ab07c namespace: kserve-ci-e2e-test resourceVersion: "27948" uid: 787db124-596a-4038-aa04-70a289ec65ec reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-760ed namespace: kserve-ci-e2e-test resourceVersion: "28504" uid: b208670c-5c90-46ad-86fb-86d6a92a8f5d kind: Event lastTimestamp: "2026-04-23T18:34:36Z" message: InferenceService [error-404-isvc-760ed] is Ready metadata: creationTimestamp: "2026-04-23T18:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:36Z" name: error-404-isvc-760ed.18a9102454314ca3 namespace: kserve-ci-e2e-test resourceVersion: "28505" uid: 63f50d5f-f5d1-4133-8702-a94a37ce42b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:23:09.042790Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22315" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-83978-predictor-775f4c4947-v4grj to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a90f8456ec54c6 namespace: kserve-ci-e2e-test resourceVersion: "22319" uid: a9217e23-59ce-4805-965b-64341b170cce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22325" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a90f84711551dd namespace: kserve-ci-e2e-test resourceVersion: "22340" uid: 66f55a2e-1ad1-45e6-b2c8-ed6acca9c033 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a90f847353e185 namespace: kserve-ci-e2e-test resourceVersion: "22342" uid: 24e7feee-f20e-4ac7-83ea-7e678c20c082 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a90f8475a842bc namespace: kserve-ci-e2e-test resourceVersion: "22345" uid: 99ceeee9-0376-4835-a8d3-8d72dc8f8cdb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a90f84764168df namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: aca859c0-8538-4fa9-86f9-544f7adc4d35 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a90f84764ee195 namespace: kserve-ci-e2e-test resourceVersion: "22347" uid: af9d0cc0-1aca-4e97-837d-5d34ae739231 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a90f84794cb5a0 namespace: kserve-ci-e2e-test resourceVersion: "22349" uid: 70da2829-c102-4db2-ad1f-cc9a6961e68f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a90f8479c4139d namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: c3caa057-111f-4895-91de-9636e30f9133 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:32:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:25Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a90f84881bdb47 namespace: kserve-ci-e2e-test resourceVersion: "27092" uid: 7d72ce83-2e59-4495-9ad0-0d6c4347fec2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:32:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:23Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a910058c6c3e27 namespace: kserve-ci-e2e-test resourceVersion: "27004" uid: 93bcd748-9303-4f33-997f-27d2addbc4af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:32:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:23Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a910058c6ccbb9 namespace: kserve-ci-e2e-test resourceVersion: "27005" uid: 43634313-4c54-45d1-8569-036bae907545 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83978-predictor-775f4c4947-v4grj namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: f49c56ec-2bbb-4583-ae1f-9c83bd399228 kind: Event lastTimestamp: "2026-04-23T18:32:25Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:25Z" name: error-404-isvc-83978-predictor-775f4c4947-v4grj.18a91005fca64dfc namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: e4c6b6b9-5ff3-46d8-bf71-1384bced00b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-83978-predictor-775f4c4947 namespace: kserve-ci-e2e-test resourceVersion: "22308" uid: 964cd213-71b9-469f-a5bc-9ae14ef1dcaf kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: 'Created pod: error-404-isvc-83978-predictor-775f4c4947-v4grj' metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor-775f4c4947.18a90f845677396f namespace: kserve-ci-e2e-test resourceVersion: "22316" uid: c2eb9dbe-b75f-41fe-9262-5627e0bab2b9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22306" uid: 8a972099-9b67-42c1-a559-0ac4eff24680 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Scaled up replica set error-404-isvc-83978-predictor-775f4c4947 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978-predictor.18a90f8454d7ba4e namespace: kserve-ci-e2e-test resourceVersion: "22310" uid: deeb7424-c76f-42d0-b419-40e117a3f3e6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22307" uid: 7ba29a88-c473-431e-bdcf-9fd85e6c0ad9 kind: Event lastTimestamp: "2026-04-23T18:23:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:39Z" name: error-404-isvc-83978-predictor.18a90f87d34b563a namespace: kserve-ci-e2e-test resourceVersion: "22615" uid: d3aff3a4-139c-4014-b537-49fd734eacde reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22307" uid: 7ba29a88-c473-431e-bdcf-9fd85e6c0ad9 kind: Event lastTimestamp: "2026-04-23T18:23:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:39Z" name: error-404-isvc-83978-predictor.18a90f87d3badd52 namespace: kserve-ci-e2e-test resourceVersion: "22616" uid: 33e7c4b6-53d5-4b7e-87b3-de200c8f7f96 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:23:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22495" uid: 7ba29a88-c473-431e-bdcf-9fd85e6c0ad9 kind: Event lastTimestamp: "2026-04-23T18:24:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:39Z" name: error-404-isvc-83978-predictor.18a90f8ed1e7a18f namespace: kserve-ci-e2e-test resourceVersion: "23533" uid: ffd675f5-b426-471e-8ce5-196c632f6718 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:23:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22495" uid: 7ba29a88-c473-431e-bdcf-9fd85e6c0ad9 kind: Event lastTimestamp: "2026-04-23T18:24:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:39Z" name: error-404-isvc-83978-predictor.18a90f8ed25b92bb namespace: kserve-ci-e2e-test resourceVersion: "23534" uid: 067ce185-923f-48e9-9700-bfcb2989b186 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-83978 namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: d69932d9-4150-453d-9000-3dbe5ef26570 kind: Event lastTimestamp: "2026-04-23T18:24:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:06Z" name: error-404-isvc-83978.18a90f8454a8258e namespace: kserve-ci-e2e-test resourceVersion: "23033" uid: a8633926-59b5-402e-a3ab-c517a42e6c51 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-83978 namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: d69932d9-4150-453d-9000-3dbe5ef26570 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-83978": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-83978": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978.18a90f8455225901 namespace: kserve-ci-e2e-test resourceVersion: "22312" uid: 46dda50e-a698-49c8-bbb0-7b5a21753e13 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-83978 namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: d69932d9-4150-453d-9000-3dbe5ef26570 kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-83978": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:09Z" name: error-404-isvc-83978.18a90f845522b02f namespace: kserve-ci-e2e-test resourceVersion: "22313" uid: ea247cdf-6264-4478-9e97-6fbc6423a6e8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-83978 namespace: kserve-ci-e2e-test resourceVersion: "23025" uid: d69932d9-4150-453d-9000-3dbe5ef26570 kind: Event lastTimestamp: "2026-04-23T18:24:06Z" message: InferenceService [error-404-isvc-83978] is Ready metadata: creationTimestamp: "2026-04-23T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:06Z" name: error-404-isvc-83978.18a90f91a8e4ce86 namespace: kserve-ci-e2e-test resourceVersion: "23028" uid: 5e17325b-49ec-4331-ae30-3ede49155692 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:19.700985Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17547" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:19Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90efb1f6a0fc8 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: da6ad2e4-d494-46bb-be37-a03e1e415101 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17566" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:20Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90efb3960d6c2 namespace: kserve-ci-e2e-test resourceVersion: "17577" uid: 73b5dc10-e78c-44b4-8830-63669b6be720 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90efb3ae8c00d namespace: kserve-ci-e2e-test resourceVersion: "17582" uid: a3727311-a4f6-4c1a-a604-d1c9a463b401 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90efb3d582fe9 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: de26a140-4f79-4b20-a83d-b4eee96c2785 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90efb3dc77708 namespace: kserve-ci-e2e-test resourceVersion: "17586" uid: a0eff31d-48a8-4a33-9eb1-65026124030a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90efb3dd4a864 namespace: kserve-ci-e2e-test resourceVersion: "17587" uid: c08f3b4c-ab46-4aad-8052-2bc35cf993bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90efb40918336 namespace: kserve-ci-e2e-test resourceVersion: "17588" uid: 97ff0198-db9c-4aff-a5b7-48c10a851051 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90efb4133134c namespace: kserve-ci-e2e-test resourceVersion: "17589" uid: 7fb15dfd-a5ea-41af-9dca-5a09a5038ef7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:22:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:47Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90efba01c4148 namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: d1829145-ade7-417f-8b63-7dc4052644ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:44Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90f7ea29fd141 namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: 4f8312ed-b04b-46e2-b59d-0a2f174d15d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:44Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90f7ea2a2db37 namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: 89e0e189-02b3-4b2c-82fd-cd4ad072dc10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 749e09bb-9d8f-4c6f-bf1c-12b7d3f4515b kind: Event lastTimestamp: "2026-04-23T18:22:47Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:47Z" name: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8.18a90f7f6890a68e namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: dcc08dbc-5d22-41c0-8d74-004593ce9737 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-89c6b-predictor-dc9b564c7 namespace: kserve-ci-e2e-test resourceVersion: "17532" uid: 082eeb4b-1954-45b6-af34-db1c212e1730 kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: 'Created pod: error-404-isvc-89c6b-predictor-dc9b564c7-qwvv8' metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:19Z" name: error-404-isvc-89c6b-predictor-dc9b564c7.18a90efb1efb637a namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: e557a138-b599-4a23-9815-0942c1ed12de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17530" uid: 9b77df79-715f-4ec0-834b-851d93430c6b kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: Scaled up replica set error-404-isvc-89c6b-predictor-dc9b564c7 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:19Z" name: error-404-isvc-89c6b-predictor.18a90efb1d6d7634 namespace: kserve-ci-e2e-test resourceVersion: "17534" uid: d4cb99e4-cc34-4935-bce9-e0b47c179d9e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17531" uid: d17e8976-1a56-4bb3-9676-df70f89d5ead kind: Event lastTimestamp: "2026-04-23T18:14:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:34Z" name: error-404-isvc-89c6b-predictor.18a90efe9bfa0271 namespace: kserve-ci-e2e-test resourceVersion: "18306" uid: 01c5d2e5-d4ab-4df0-82e1-6967723ac985 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17531" uid: d17e8976-1a56-4bb3-9676-df70f89d5ead kind: Event lastTimestamp: "2026-04-23T18:14:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:34Z" name: error-404-isvc-89c6b-predictor.18a90efe9c745194 namespace: kserve-ci-e2e-test resourceVersion: "18307" uid: a9c2769f-8c0e-4b17-a7f7-c4d381a5981a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17754" uid: d17e8976-1a56-4bb3-9676-df70f89d5ead kind: Event lastTimestamp: "2026-04-23T18:13:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:49Z" name: error-404-isvc-89c6b-predictor.18a90f021b6136db namespace: kserve-ci-e2e-test resourceVersion: "17908" uid: 0e9e74a3-af46-4d76-aeb4-13e1807307fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17754" uid: d17e8976-1a56-4bb3-9676-df70f89d5ead kind: Event lastTimestamp: "2026-04-23T18:13:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:49Z" name: error-404-isvc-89c6b-predictor.18a90f021bcf1d9c namespace: kserve-ci-e2e-test resourceVersion: "17910" uid: 36e1d8f2-24d9-4f50-aeba-9726be808a1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89c6b namespace: kserve-ci-e2e-test resourceVersion: "17501" uid: 6481cb72-3312-412f-a6e4-54af1c3a6ec7 kind: Event lastTimestamp: "2026-04-23T18:14:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:18Z" name: error-404-isvc-89c6b.18a90efb1d33546d namespace: kserve-ci-e2e-test resourceVersion: "18145" uid: 0f4a3494-f205-417b-a5e8-552f26bd53ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89c6b namespace: kserve-ci-e2e-test resourceVersion: "17501" uid: 6481cb72-3312-412f-a6e4-54af1c3a6ec7 kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: 'Failed to update status for InferenceService "error-404-isvc-89c6b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-89c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:19Z" name: error-404-isvc-89c6b.18a90efb1d7cf90b namespace: kserve-ci-e2e-test resourceVersion: "17536" uid: 700c9883-4ced-4a4e-9ba4-9d9eadeac696 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89c6b namespace: kserve-ci-e2e-test resourceVersion: "17501" uid: 6481cb72-3312-412f-a6e4-54af1c3a6ec7 kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-89c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:19Z" name: error-404-isvc-89c6b.18a90efb1d7d5a62 namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 15e152e5-6b60-4783-af0e-a7ad5dfb9c98 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18139" uid: 6481cb72-3312-412f-a6e4-54af1c3a6ec7 kind: Event lastTimestamp: "2026-04-23T18:14:17Z" message: InferenceService [error-404-isvc-89c6b] is Ready metadata: creationTimestamp: "2026-04-23T18:14:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:17Z" name: error-404-isvc-89c6b.18a90f08b0a8b9a8 namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: 0e46704e-246d-4083-abf4-974a1a276679 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:22:44.659511Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:22:44Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f7ea9910eda namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: 2bf10de6-ae3b-4d6e-888c-fc8bdc9d664a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:22:45Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f7ec35ed7d3 namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: 9413cb5a-ea92-43bd-b01a-857e1072b222 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f7ec4a936d3 namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: 6083f1e3-c65a-492b-958e-aeecc1da4f69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f7ec7a945c1 namespace: kserve-ci-e2e-test resourceVersion: "21940" uid: f4d4e098-4966-44e2-b5fc-cbf85c1fb5c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f7ec8227cd3 namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: 5732634b-5fba-411d-8bca-744a1c4857ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f7ec83166b7 namespace: kserve-ci-e2e-test resourceVersion: "21942" uid: 389a01b4-cb3f-49f9-964c-5c22fd70939c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f7ecaf9b381 namespace: kserve-ci-e2e-test resourceVersion: "21943" uid: b96f7232-d078-43eb-81b6-6c7c485bccd4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f7ecb6d3b75 namespace: kserve-ci-e2e-test resourceVersion: "21944" uid: f70543a8-f943-4205-b2d4-0c0204fb6e60 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:23:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:31Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f7eebcd346f namespace: kserve-ci-e2e-test resourceVersion: "22560" uid: 3ce14a0c-f871-42bc-8809-94a998ada52c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:04Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f9153469387 namespace: kserve-ci-e2e-test resourceVersion: "22961" uid: 8078e952-f4da-47b5-97d5-cd37395da91a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:04Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f9153479757 namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 03c20181-b350-41c2-b6b2-185391956c03 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 5227b4d9-66cc-41bb-959c-f3591b46323d kind: Event lastTimestamp: "2026-04-23T18:24:06Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:06Z" name: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb.18a90f91c80d7c9a namespace: kserve-ci-e2e-test resourceVersion: "23037" uid: 7defc556-3b5e-46b0-aa78-003d7ee99bca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-89ff7-predictor-56ff5477b5 namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: 90efd2eb-6354-4809-94cb-943d6f54ed25 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: 'Created pod: error-404-isvc-89ff7-predictor-56ff5477b5-w7ffb' metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:44Z" name: error-404-isvc-89ff7-predictor-56ff5477b5.18a90f7ea8c1f007 namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: bfc81a57-f179-4c12-8e9a-33f5ad068f27 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 6b159e92-f20e-44a4-a61a-57a0e9757459 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: Scaled up replica set error-404-isvc-89ff7-predictor-56ff5477b5 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:44Z" name: error-404-isvc-89ff7-predictor.18a90f7ea6b3a6b2 namespace: kserve-ci-e2e-test resourceVersion: "21912" uid: d6d39f95-3302-4561-8435-2e01e6536e9c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: a5c0476c-8a83-4599-8ec6-0030ebc7971a kind: Event lastTimestamp: "2026-04-23T18:23:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:14Z" name: error-404-isvc-89ff7-predictor.18a90f8224f0afba namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: ed210c5e-c231-4d59-b677-5fa8eba819bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: a5c0476c-8a83-4599-8ec6-0030ebc7971a kind: Event lastTimestamp: "2026-04-23T18:23:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:14Z" name: error-404-isvc-89ff7-predictor.18a90f82255ba34e namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: f367a223-a543-4f6b-8429-acc153d9d9fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:23:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22140" uid: a5c0476c-8a83-4599-8ec6-0030ebc7971a kind: Event lastTimestamp: "2026-04-23T18:23:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:59Z" name: error-404-isvc-89ff7-predictor.18a90f8923242e3c namespace: kserve-ci-e2e-test resourceVersion: "22838" uid: 99190827-5648-4e8d-89ce-f43785e971da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:23:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22140" uid: a5c0476c-8a83-4599-8ec6-0030ebc7971a kind: Event lastTimestamp: "2026-04-23T18:23:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:59Z" name: error-404-isvc-89ff7-predictor.18a90f8923edaece namespace: kserve-ci-e2e-test resourceVersion: "22839" uid: 9d676ced-9fb8-4056-a8b1-d71cde377535 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a661f0e3-fcfb-48df-b73f-f877320cb327 kind: Event lastTimestamp: "2026-04-23T18:23:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:41Z" name: error-404-isvc-89ff7.18a90f7ea64177da namespace: kserve-ci-e2e-test resourceVersion: "22647" uid: 521e1e05-5473-4189-a368-c755ee32ee34 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a661f0e3-fcfb-48df-b73f-f877320cb327 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-89ff7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-89ff7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:44Z" name: error-404-isvc-89ff7.18a90f7ea6ad3c35 namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: 3513965e-3c42-400f-b6c3-29112fb27d73 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: a661f0e3-fcfb-48df-b73f-f877320cb327 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-89ff7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:44Z" name: error-404-isvc-89ff7.18a90f7ea6ad905f namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: ef95ca79-2373-4f57-87ad-51925ecc22ca reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "22643" uid: a661f0e3-fcfb-48df-b73f-f877320cb327 kind: Event lastTimestamp: "2026-04-23T18:23:41Z" message: InferenceService [error-404-isvc-89ff7] is Ready metadata: creationTimestamp: "2026-04-23T18:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:41Z" name: error-404-isvc-89ff7.18a90f8bfaa12b9e namespace: kserve-ci-e2e-test resourceVersion: "22645" uid: c6e29566-45a1-42e0-a506-f97a483a4eb0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:10:59.845963Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-be430-predictor-6bcf86bc7-2wc2j to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:10:59Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90eda8f67b96f namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: fffed5ea-01f1-406e-bee5-b0e1b72f3172 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:11:00Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:11:00Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edaa9071515 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: efb2d3cc-91e7-4485-b5fc-df658936e96a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:11:00Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:00Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edaaa59c381 namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: b20faf4d-0ac2-4b41-a747-04f08f782f5a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" in 18.271s (18.271s including waiting). Image size: 1336608873 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edeeb6ca5d5 namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: cd3e73fc-d4c7-422d-98a2-c5d6d1e0e914 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edeefd45678 namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: df788e80-aefb-4376-a476-c1c8b95c987a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edef05d2857 namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 12e9e728-c388-4dfa-9f08-86c3b2991fac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edef06cd621 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 44db05e4-33ce-4f13-a3d1-9a16aa2f9bfd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:11:21Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.585s (2.585s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:21Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edf8a8ebbbf namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 7f2266e7-e253-4217-9403-b97a78a4dbc9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:11:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:21Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edf8e986596 namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: b66c4fbd-cc27-4b7f-9d3c-473b8d54f1df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:11:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:21Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edf8f108238 namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: d7346e70-77ca-47ca-a4a2-bad548e4349d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:12:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:08Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90edfd2239133 namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: 2c00a056-6cfb-4e17-8d01-cb7ea90636ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:33Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90ef0691d9b62 namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: f5c245b1-73cd-48ba-b0c9-6078e2eadbd6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 168fefc9-6f1d-42a3-8e99-e9ae9bee9760 kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:33Z" name: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j.18a90ef0691e90fe namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: 352238e2-f64e-4f28-a4a3-f70824e55f83 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-be430-predictor-6bcf86bc7 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: c7433fdf-8a2b-4e59-b8ed-61e6c5a42574 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: 'Created pod: error-404-isvc-be430-predictor-6bcf86bc7-2wc2j' metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:10:59Z" name: error-404-isvc-be430-predictor-6bcf86bc7.18a90eda8ee35c64 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: f3715a3a-1f35-4645-9d81-6cbefb1c1266 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: 2abf9771-45a3-45af-9de6-8797b846e55e kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: Scaled up replica set error-404-isvc-be430-predictor-6bcf86bc7 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:10:59Z" name: error-404-isvc-be430-predictor.18a90eda893cb9e6 namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: bca11015-b8e4-48ec-b56a-48151e1752b7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:11:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: bc1f136f-8c08-4dac-a186-d79089d1ad0c kind: Event lastTimestamp: "2026-04-23T18:11:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:29Z" name: error-404-isvc-be430-predictor.18a90ede082a988d namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: 4321e41b-a4a6-4564-9877-70b468e162ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:11:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: bc1f136f-8c08-4dac-a186-d79089d1ad0c kind: Event lastTimestamp: "2026-04-23T18:11:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:29Z" name: error-404-isvc-be430-predictor.18a90ede08a5daef namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 6c67e9d2-f3ff-493a-8287-3b34d1585777 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:11:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: bc1f136f-8c08-4dac-a186-d79089d1ad0c kind: Event lastTimestamp: "2026-04-23T18:12:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:29Z" name: error-404-isvc-be430-predictor.18a90ee506239d1d namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: f7919689-002b-4324-9aa7-e8513647a192 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:11:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: bc1f136f-8c08-4dac-a186-d79089d1ad0c kind: Event lastTimestamp: "2026-04-23T18:12:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:29Z" name: error-404-isvc-be430-predictor.18a90ee50674461b namespace: kserve-ci-e2e-test resourceVersion: "16834" uid: 577dbdde-2b79-4605-aa8c-526eb3d54ac0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be430 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: d47560fc-ad40-43df-9179-914ada6ab82b kind: Event lastTimestamp: "2026-04-23T18:12:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:18Z" name: error-404-isvc-be430.18a90eda897a74e3 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 8503d09e-b719-4e2c-8776-71bc056dcb0e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be430 namespace: kserve-ci-e2e-test resourceVersion: "16693" uid: d47560fc-ad40-43df-9179-914ada6ab82b kind: Event lastTimestamp: "2026-04-23T18:12:18Z" message: InferenceService [error-404-isvc-be430] is Ready metadata: creationTimestamp: "2026-04-23T18:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:18Z" name: error-404-isvc-be430.18a90eece0df2629 namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: a76e5b2b-9522-417a-942a-2ccdd8bceba4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:33:29.804296Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27727" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-febb8-predictor-545595fd9f-5vcsh to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:33:29Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a91014df314b5a namespace: kserve-ci-e2e-test resourceVersion: "27731" uid: 858c8825-55c3-45a9-9666-71119eba6cb2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-febb8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:29Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a91014ea97eb7b namespace: kserve-ci-e2e-test resourceVersion: "27743" uid: de96751f-a9d4-45a7-af2d-caf443b8ccfa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27735" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:33:30Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a910151cd70942 namespace: kserve-ci-e2e-test resourceVersion: "27763" uid: dea786bd-903c-4eb6-93ae-21b4ac252a3e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a910151ec93c2b namespace: kserve-ci-e2e-test resourceVersion: "27765" uid: b6ae2f4e-9775-47fb-81c7-a2c6be4a9606 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a9101521e0f3b8 namespace: kserve-ci-e2e-test resourceVersion: "27768" uid: 9675c9e7-f089-41c5-990e-afd009ebed3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a91015225a4af7 namespace: kserve-ci-e2e-test resourceVersion: "27769" uid: 907af56e-da49-46d6-b702-ab59336100e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a9101522676779 namespace: kserve-ci-e2e-test resourceVersion: "27770" uid: 54834aa3-3e4a-4fe5-9a30-53896409af9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a9101526195068 namespace: kserve-ci-e2e-test resourceVersion: "27771" uid: e4060a06-6a3c-4e3b-a1b8-6005ce7c4ad2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:33:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:31Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a9101526914905 namespace: kserve-ci-e2e-test resourceVersion: "27772" uid: b88033a7-ff10-48a5-977c-ed752605c6e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:34:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:08Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a91015657efe5c namespace: kserve-ci-e2e-test resourceVersion: "28245" uid: e6dd43da-acfe-428f-be4d-ebf8022b2756 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:50:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:50:59Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a9110935c1daa4 namespace: kserve-ci-e2e-test resourceVersion: "35399" uid: d1052b5b-1782-41dd-93fd-57d345d22058 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 2e88b719-da61-49fb-b16f-981e4d856938 kind: Event lastTimestamp: "2026-04-23T18:50:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:50:59Z" name: error-404-isvc-febb8-predictor-545595fd9f-5vcsh.18a9110935c25945 namespace: kserve-ci-e2e-test resourceVersion: "35400" uid: c2292a41-5fd8-4781-9d5a-33a0d1020608 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-febb8-predictor-545595fd9f namespace: kserve-ci-e2e-test resourceVersion: "27720" uid: 85c4125a-0b09-484f-97fd-85cb8866d84c kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: 'Created pod: error-404-isvc-febb8-predictor-545595fd9f-5vcsh' metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:29Z" name: error-404-isvc-febb8-predictor-545595fd9f.18a91014de4c4f44 namespace: kserve-ci-e2e-test resourceVersion: "27728" uid: 4331a7ab-ccd2-419d-bfbe-699e0912c964 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "27718" uid: cf9c80e3-d058-4001-9bed-960ae498657f kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: Scaled up replica set error-404-isvc-febb8-predictor-545595fd9f from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:29Z" name: error-404-isvc-febb8-predictor.18a91014dc061083 namespace: kserve-ci-e2e-test resourceVersion: "27722" uid: 1f18d3c6-547a-4b3e-9d6c-9d3ecf25ca11 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:33:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "27719" uid: ce42776b-b79e-40d7-bf5b-ac04a4253b6b kind: Event lastTimestamp: "2026-04-23T18:34:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:33:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:14Z" name: error-404-isvc-febb8-predictor.18a910185a8628b2 namespace: kserve-ci-e2e-test resourceVersion: "28307" uid: 185f4b58-776d-44a7-8c68-df0aa564636b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:33:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "27719" uid: ce42776b-b79e-40d7-bf5b-ac04a4253b6b kind: Event lastTimestamp: "2026-04-23T18:34:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:33:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:14Z" name: error-404-isvc-febb8-predictor.18a910185ae2b0c1 namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 94cb0ddd-b5da-4257-a235-c93aabbfe9a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:33:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "28055" uid: ce42776b-b79e-40d7-bf5b-ac04a4253b6b kind: Event lastTimestamp: "2026-04-23T18:34:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:33:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:44Z" name: error-404-isvc-febb8-predictor.18a9101bd9937d2e namespace: kserve-ci-e2e-test resourceVersion: "28650" uid: 867449f7-744b-4852-829e-3552903b263d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:33:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "28055" uid: ce42776b-b79e-40d7-bf5b-ac04a4253b6b kind: Event lastTimestamp: "2026-04-23T18:34:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:33:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:44Z" name: error-404-isvc-febb8-predictor.18a9101bda0470da namespace: kserve-ci-e2e-test resourceVersion: "28651" uid: 5935a604-e15f-44a7-a1cb-535a73ae1664 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-febb8 namespace: kserve-ci-e2e-test resourceVersion: "27695" uid: be57c472-fdfa-48ed-bd52-9705cacc004f kind: Event lastTimestamp: "2026-04-23T18:34:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:18Z" name: error-404-isvc-febb8.18a91014dbdfa29c namespace: kserve-ci-e2e-test resourceVersion: "28337" uid: 9debb909-4fe0-45fc-9b21-9e7aef66b196 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-febb8 namespace: kserve-ci-e2e-test resourceVersion: "27695" uid: be57c472-fdfa-48ed-bd52-9705cacc004f kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-febb8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-febb8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:29Z" name: error-404-isvc-febb8.18a91014dc430ba4 namespace: kserve-ci-e2e-test resourceVersion: "27724" uid: b47ecb38-dc01-4ad8-bff0-8a47e7217ad4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-febb8 namespace: kserve-ci-e2e-test resourceVersion: "27695" uid: be57c472-fdfa-48ed-bd52-9705cacc004f kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-febb8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:29Z" name: error-404-isvc-febb8.18a91014dc435b00 namespace: kserve-ci-e2e-test resourceVersion: "27725" uid: 2e5d9b67-5cdf-4152-9b50-62f821c832b6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-febb8 namespace: kserve-ci-e2e-test resourceVersion: "28335" uid: be57c472-fdfa-48ed-bd52-9705cacc004f kind: Event lastTimestamp: "2026-04-23T18:34:18Z" message: InferenceService [error-404-isvc-febb8] is Ready metadata: creationTimestamp: "2026-04-23T18:34:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:18Z" name: error-404-isvc-febb8.18a9102021c0ca56 namespace: kserve-ci-e2e-test resourceVersion: "28336" uid: 06c522fa-0840-4b4c-8639-09e7d5035585 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:10:59.458769Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-588b77b98-h9jfg to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:10:59Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90eda7853a21d namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: c8c312c7-e283-4e0c-8876-10503107d97d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:10:59Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90eda917f4786 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: af704fe6-1bb8-4f33-bd1c-532f07008daa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:8fb08be5bf3c908942a74ae20f5d98202838ebfb02dff98ba5e436bfa0ba8437" metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:10:59Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90eda92be01b4 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: dc3b0bce-7f4d-45f1-9910-fe2d93dd9d1a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:8fb08be5bf3c908942a74ae20f5d98202838ebfb02dff98ba5e436bfa0ba8437" in 4.296s (4.296s including waiting). Image size: 301160889 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:04Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90edb92cf71f5 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: d74f763c-feea-4131-afcd-adcb9230fd5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:04Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90edb9627fd53 namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: 6fcf6dd3-34ab-4c2b-8bd2-13d5e2cf141d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:04Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90edb969c6582 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 95d1da48-e946-4c80-a0e1-c1fe80e7834d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:08Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1419" metadata: creationTimestamp: "2026-04-23T18:11:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:08Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90edc9274bded namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: caf4e297-0e10-4373-b3d1-4407db104075 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:32Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1419" in 24.441s (24.441s including waiting). Image size: 1562904500 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:32Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90ee2434c0f31 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 65051f47-ea24-4abe-bfbc-9b1f7629c949 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:33Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90ee2491f750e namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: e4b30545-29e0-4f76-914d-7642ccfb04da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:33Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90ee24a33b72b namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: a0175eb7-d337-41a0-9f2b-3035bf5a4816 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:33Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:33Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90ee24a402c60 namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: cd2e008e-4e04-4460-bae5-0cc76018e231 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:36Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.308s (3.308s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:36Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90ee30f730838 namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 6d240408-cc1e-4350-b6ea-0275e9884365 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:36Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90ee315352bbf namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 748fe3ae-ac15-431a-8ca9-885176f42bc0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:11:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:36Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90ee31703dfe6 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 00815735-1c90-4330-b285-cd6f51ef46f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:13:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:22Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90ee31e0a4f35 namespace: kserve-ci-e2e-test resourceVersion: "17623" uid: df8343bd-454f-449e-81d4-6f5d1fca441a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:19Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90efb189b18a1 namespace: kserve-ci-e2e-test resourceVersion: "17500" uid: 7bd7ae5f-a2f0-49f5-8285-0e078d855ca0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:19Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90efb189bbc3b namespace: kserve-ci-e2e-test resourceVersion: "17503" uid: 27423f52-64d6-4ec4-8c71-a9a6c6b1afb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 04269895-41e9-4b40-a921-26abf7648545 kind: Event lastTimestamp: "2026-04-23T18:13:22Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:22Z" name: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg.18a90efbcc50d0e4 namespace: kserve-ci-e2e-test resourceVersion: "17622" uid: 86a762f6-4ca8-429d-87c7-8d3032cfb4d9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-588b77b98 namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: 1cc1c98f-b8e2-417e-ad9c-af1364e13547 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-588b77b98-h9jfg' metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:10:59Z" name: isvc-sklearn-graph-1-predictor-588b77b98.18a90eda77df560b namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: f1afb1a9-2b0d-469e-99dc-9fad39ee6e59 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: a90ce547-918d-4b94-a48b-93b2f9124454 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-588b77b98 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:10:59Z" name: isvc-sklearn-graph-1-predictor.18a90eda70db53aa namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: 4514de1a-24a5-4cd3-b35c-2e4f1a3f3fe2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: 124d30f3-eb4e-45fa-8d71-615565fef1b8 kind: Event lastTimestamp: "2026-04-23T18:11:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:44Z" name: isvc-sklearn-graph-1-predictor.18a90eddf5ef880c namespace: kserve-ci-e2e-test resourceVersion: "16395" uid: 6a918842-2ea6-49e6-add6-570f64345891 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: 124d30f3-eb4e-45fa-8d71-615565fef1b8 kind: Event lastTimestamp: "2026-04-23T18:11:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:44Z" name: isvc-sklearn-graph-1-predictor.18a90eddf65aaabc namespace: kserve-ci-e2e-test resourceVersion: "16396" uid: e592461b-0141-4057-8e7d-0144f01a833d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:11:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 124d30f3-eb4e-45fa-8d71-615565fef1b8 kind: Event lastTimestamp: "2026-04-23T18:13:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:14Z" name: isvc-sklearn-graph-1-predictor.18a90ee87335f6e0 namespace: kserve-ci-e2e-test resourceVersion: "17392" uid: e5395356-ced2-4fe4-a889-8b485378d917 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:11:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 124d30f3-eb4e-45fa-8d71-615565fef1b8 kind: Event lastTimestamp: "2026-04-23T18:13:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:14Z" name: isvc-sklearn-graph-1-predictor.18a90ee873b003f6 namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: 87ee5c0f-e212-4091-a996-b28c7b5e24c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: f2bd14c5-7348-47cb-97f4-d845365261a4 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:42Z" name: isvc-sklearn-graph-1.18a90eda76c6a267 namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: d506c062-a9b2-4761-87fa-47ee5e4dde69 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "17088" uid: f2bd14c5-7348-47cb-97f4-d845365261a4 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:42Z" name: isvc-sklearn-graph-1.18a90ef281cb5b7e namespace: kserve-ci-e2e-test resourceVersion: "17089" uid: 3c10fa30-5687-4d6f-b114-accae4f32cb8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:11:00.609051Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:11:00Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90edabce395df namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: e2ee7775-6e92-45b2-959c-450f1ec057d0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:01Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:11:01Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90edad7cac108 namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: d6d54f6f-fcf3-4a15-af2d-6869b7e6e6ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:01Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:8fb08be5bf3c908942a74ae20f5d98202838ebfb02dff98ba5e436bfa0ba8437" metadata: creationTimestamp: "2026-04-23T18:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:01Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90edad9c7f1f4 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: cd7ce683-ce90-4b78-b143-f81bdc3f2019 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:8fb08be5bf3c908942a74ae20f5d98202838ebfb02dff98ba5e436bfa0ba8437" in 14.184s (14.184s including waiting). Image size: 301160889 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:15Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90ede273bda35 namespace: kserve-ci-e2e-test resourceVersion: "16108" uid: 037078cb-278b-456c-bcfe-dd30bf78a57e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90edeecd344f6 namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 7a21a091-a4e9-4e15-b96e-81525726e33b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90edeed4ed825 namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: a1842c5f-84e0-431a-b36f-e627ff2cea63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1419" metadata: creationTimestamp: "2026-04-23T18:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:23Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90ee00df5aec1 namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 45a2ac02-1f28-4908-b80a-f25323a6c78a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:30Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1419" in 6.79s (6.79s including waiting). Image size: 1562904500 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:30Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90ee1a2b13486 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: e099255c-d68e-4fb9-a316-1220ea9dd420 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:30Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90ee1a8dbf058 namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 0de201da-87ff-4e1e-9dfe-d2a2b188ca10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:30Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90ee1aa426b50 namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 18cdd2f0-b503-4b03-970d-68ae48bc6de3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:30Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90ee1aa645f5c namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: 467160cc-3301-4285-9aec-13366599a65e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:30Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90ee1af9af8dd namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: d48e6449-2fca-4a42-892d-5379a400af28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:11:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:30Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90ee1b0eb3119 namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: 9f5bc548-a0de-447f-8b04-39a22e2a9326 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:12:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:38Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90ee22868ba2e namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: 66c9eb0d-2137-48ce-9a73-6bcdfaaecbae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:19Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90efb1ffc541b namespace: kserve-ci-e2e-test resourceVersion: "17560" uid: 467d93fb-8178-40f3-82fc-56f42982cfbd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:19Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90efb1ffce22d namespace: kserve-ci-e2e-test resourceVersion: "17565" uid: 37d12a40-2945-44a0-bf5a-e51b3756f49a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c7b621e1-9f6f-4cdf-99ad-766c2ff177fa kind: Event lastTimestamp: "2026-04-23T18:13:23Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:23Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9.18a90efc00b000a4 namespace: kserve-ci-e2e-test resourceVersion: "17635" uid: 6dc7dd2e-12c4-4704-a47e-0cc68f450d56 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7d9dcd4478 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: fbbb1ca4-6819-4108-80be-c6889f4e2fd3 kind: Event lastTimestamp: "2026-04-23T18:11:00Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7d9dcd4478-xpgv9' metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:00Z" name: isvc-sklearn-graph-2-predictor-7d9dcd4478.18a90edabc54347a namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: d60ec4a0-a1c7-445a-9843-9884022efc26 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 4e80bf01-0c7e-498d-98ad-2afc44f015fd kind: Event lastTimestamp: "2026-04-23T18:11:00Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7d9dcd4478 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:00Z" name: isvc-sklearn-graph-2-predictor.18a90edaa2e15581 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 156ef7b0-02d9-448a-af39-6e9807c40ee9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: a2b166f7-3bcb-408b-959b-3e305a54d332 kind: Event lastTimestamp: "2026-04-23T18:11:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:45Z" name: isvc-sklearn-graph-2-predictor.18a90ede21590d32 namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: 6d0c1918-3915-408a-9eaa-d4b78e0203f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: a2b166f7-3bcb-408b-959b-3e305a54d332 kind: Event lastTimestamp: "2026-04-23T18:11:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:45Z" name: isvc-sklearn-graph-2-predictor.18a90ede21bf2aed namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: 3b3c139a-af91-4a03-bc09-92d35825b780 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: a2b166f7-3bcb-408b-959b-3e305a54d332 kind: Event lastTimestamp: "2026-04-23T18:13:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:15Z" name: isvc-sklearn-graph-2-predictor.18a90ee89e82d450 namespace: kserve-ci-e2e-test resourceVersion: "17401" uid: 7c0acba4-c79d-4baf-a9ec-6965c38ab616 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: a2b166f7-3bcb-408b-959b-3e305a54d332 kind: Event lastTimestamp: "2026-04-23T18:13:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:15Z" name: isvc-sklearn-graph-2-predictor.18a90ee89ef5aa78 namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: 7d29de54-70ae-463d-b49b-c75fcbab0c51 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 59c03238-6473-4ffb-bd4b-0649a10f3964 kind: Event lastTimestamp: "2026-04-23T18:12:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:48Z" name: isvc-sklearn-graph-2.18a90edaa2a924b5 namespace: kserve-ci-e2e-test resourceVersion: "17149" uid: cdcbf9b8-538a-4055-b82e-9e3c67934867 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: 59c03238-6473-4ffb-bd4b-0649a10f3964 kind: Event lastTimestamp: "2026-04-23T18:12:48Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-23T18:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:48Z" name: isvc-sklearn-graph-2.18a90ef3e0533411 namespace: kserve-ci-e2e-test resourceVersion: "17147" uid: 50e6fc98-32dd-4c21-a5a9-d6c88a49583f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:10:59.723419Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-htg4m to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:10:59Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90eda8819d9ae namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 2ada9b1e-1825-4b83-9990-6316d6eeb44f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:00Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:11:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90edaa144c796 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 1650fb0c-89d1-4852-b7f5-fdb3adde9fec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:00Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:8fb08be5bf3c908942a74ae20f5d98202838ebfb02dff98ba5e436bfa0ba8437" metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90edaa2d49ec6 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 96b69d31-fca0-4eeb-a508-339d2c828594 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:8fb08be5bf3c908942a74ae20f5d98202838ebfb02dff98ba5e436bfa0ba8437" in 4.032s (4.032s including waiting). Image size: 301160889 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:04Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90edb9335dcbe namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 760389c8-f953-43ac-b3ac-f9f26c9e60ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T18:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:04Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90edb9634460d namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: c4b5776e-4444-46ee-8b03-18bf1ef2cf05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T18:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:04Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90edb96be6e22 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: b6ddff4f-eede-42f1-80d5-1b44f7209af1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:08Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-23T18:11:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:08Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90edc926416ef namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: f041ebad-89e1-44f2-bb63-a36f0851510d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:33Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 24.827s (24.827s including waiting). Image size: 1306414326 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:33Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90ee25a3adea3 namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 832f80d6-f75a-43f9-b34d-05cb0b7098a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:33Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90ee260d2adb8 namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: f056e285-eff0-4fe4-bea5-5aba3a72595e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:33Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90ee262132239 namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: b981fdad-a0fe-4d98-af3f-c0a542e1d396 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:33Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:33Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90ee2621e8bca namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: dbefbda8-beab-4821-b1f3-a6ebe668f0f7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:36Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.899s (2.899s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:36Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90ee30ef0f0a2 namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: b07d4a4e-ce43-4fb7-9115-ec82e03cccee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:36Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90ee3152590ac namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: ff290040-dd79-40a0-852f-c283b59f06a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:11:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:36Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90ee317911826 namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 2b6873e7-7a11-47ea-b1cf-0e3d67af73e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:11:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:13:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:22Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90ee31df6457e namespace: kserve-ci-e2e-test resourceVersion: "17624" uid: 74e4f51e-8298-4a0e-9342-44d3dacc6e6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:19Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90efb1ba8dbb2 namespace: kserve-ci-e2e-test resourceVersion: "17523" uid: 69b1c125-bdaf-4013-9041-40a1ee6a47cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:19Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90efb1ba95b3e namespace: kserve-ci-e2e-test resourceVersion: "17525" uid: 616c60c1-2654-49a1-90d7-a2ffc90da033 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-htg4m namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: f760979b-5b29-4f7b-99b5-00538b24bd1d kind: Event lastTimestamp: "2026-04-23T18:13:22Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:22Z" name: isvc-xgboost-graph-predictor-669d8d6456-htg4m.18a90efbcc4f7bd7 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 163ed66e-3713-46c7-9c07-6d4a02c52758 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 5f3aee5c-ba68-4e82-aa61-c64d0fa81a68 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-htg4m' metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:10:59Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a90eda877085d4 namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 1d1c3eb9-8e43-4327-8c0b-bcfbf066c2e4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: 2b2e63c4-f4a9-447c-8802-e6d89efb96f4 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:10:59Z" name: isvc-xgboost-graph-predictor.18a90eda8170bc37 namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: b7c51d9a-f5d6-47b9-b46a-1d892f77921b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:11:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 048e21bf-5ba2-4e7f-869b-e84c0e763607 kind: Event lastTimestamp: "2026-04-23T18:12:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:14Z" name: isvc-xgboost-graph-predictor.18a90eddfffdddeb namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 15823440-81dc-408e-828f-f08b8ee8a225 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:11:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 048e21bf-5ba2-4e7f-869b-e84c0e763607 kind: Event lastTimestamp: "2026-04-23T18:12:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:14Z" name: isvc-xgboost-graph-predictor.18a90ede00866e88 namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: 7e6f2034-ed6a-4e2a-974c-1a04f47c5db7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:11:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 048e21bf-5ba2-4e7f-869b-e84c0e763607 kind: Event lastTimestamp: "2026-04-23T18:13:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:14Z" name: isvc-xgboost-graph-predictor.18a90ee87cf914de namespace: kserve-ci-e2e-test resourceVersion: "17397" uid: 3beea79e-e9c1-4679-ad5a-64d58e451988 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:11:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 048e21bf-5ba2-4e7f-869b-e84c0e763607 kind: Event lastTimestamp: "2026-04-23T18:13:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:14Z" name: isvc-xgboost-graph-predictor.18a90ee87d6da665 namespace: kserve-ci-e2e-test resourceVersion: "17398" uid: 75451f95-deb2-4174-b51e-01754afc3996 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: cd3e084c-0fb7-41cd-865a-adb1c200f849 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:42Z" name: isvc-xgboost-graph.18a90eda813cd043 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 196f3cf0-7a93-40c0-b743-acaab8b7d954 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "17091" uid: cd3e084c-0fb7-41cd-865a-adb1c200f849 kind: Event lastTimestamp: "2026-04-23T18:12:42Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-23T18:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:42Z" name: isvc-xgboost-graph.18a90ef2840c7b0f namespace: kserve-ci-e2e-test resourceVersion: "17092" uid: 8ce4b9f5-230a-4635-a215-95996447b468 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:09.362282Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-8698b786b7-kbqbh namespace: kserve-ci-e2e-test resourceVersion: "17329" uid: 90caca3c-a394-4fc1-b5c1-0622d17c4e62 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-8698b786b7-kbqbh to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:09Z" name: model-chainer-8698b786b7-kbqbh.18a90ef8b72df057 namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 482b2864-599d-4adc-bedf-dc51a7a32ca3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-8698b786b7-kbqbh namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 90caca3c-a394-4fc1-b5c1-0622d17c4e62 kind: Event lastTimestamp: "2026-04-23T18:13:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:09Z" name: model-chainer-8698b786b7-kbqbh.18a90ef8c288663d namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: 9c024e78-a166-46bf-85fe-6d8fc6696211 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-8698b786b7-kbqbh namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: 90caca3c-a394-4fc1-b5c1-0622d17c4e62 kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:10Z" name: model-chainer-8698b786b7-kbqbh.18a90ef8f3bf8972 namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: 33ba5982-dcfb-452e-a622-33fdf26f7d65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8698b786b7-kbqbh namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 90caca3c-a394-4fc1-b5c1-0622d17c4e62 kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:10Z" name: model-chainer-8698b786b7-kbqbh.18a90ef8f506551d namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: 1fadf5a5-6948-4a29-953f-cafda06e47cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8698b786b7-kbqbh namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 90caca3c-a394-4fc1-b5c1-0622d17c4e62 kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:10Z" name: model-chainer-8698b786b7-kbqbh.18a90ef8f7b29de8 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: c7f61b69-99f9-4883-874c-984c8bb2bed0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8698b786b7-kbqbh namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 90caca3c-a394-4fc1-b5c1-0622d17c4e62 kind: Event lastTimestamp: "2026-04-23T18:13:10Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-23T18:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:10Z" name: model-chainer-8698b786b7-kbqbh.18a90ef8f81eb480 namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: de8518aa-633a-47e9-b5e4-d47947d02627 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8698b786b7-kbqbh namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 90caca3c-a394-4fc1-b5c1-0622d17c4e62 kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:19Z" name: model-chainer-8698b786b7-kbqbh.18a90efb112a9c8a namespace: kserve-ci-e2e-test resourceVersion: "17460" uid: 633a7a77-a73b-457d-a121-e6b6b121db10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8698b786b7-kbqbh namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 90caca3c-a394-4fc1-b5c1-0622d17c4e62 kind: Event lastTimestamp: "2026-04-23T18:13:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:46Z" name: model-chainer-8698b786b7-kbqbh.18a90efba255439a namespace: kserve-ci-e2e-test resourceVersion: "17879" uid: 16e111bb-e3f1-4425-bd3f-964ec2de224e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-8698b786b7 namespace: kserve-ci-e2e-test resourceVersion: "17326" uid: 4b70460f-cea6-48b4-a6fb-243c934a01ff kind: Event lastTimestamp: "2026-04-23T18:13:09Z" message: 'Created pod: model-chainer-8698b786b7-kbqbh' metadata: creationTimestamp: "2026-04-23T18:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:09Z" name: model-chainer-8698b786b7.18a90ef8b6bfc7ff namespace: kserve-ci-e2e-test resourceVersion: "17330" uid: d35a61c5-2c45-435d-823c-222d497e411e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: ea95c37a-723b-4e90-a181-b5d71e9be636 kind: Event lastTimestamp: "2026-04-23T18:13:09Z" message: Scaled up replica set model-chainer-8698b786b7 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:09Z" name: model-chainer.18a90ef8b5fb0624 namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: f5e3f05e-e926-4ba4-b862-e1a9f5ef46a4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17340" uid: a30a37ea-03ce-4cce-9420-2500c0933a6b kind: Event lastTimestamp: "2026-04-23T18:13:09Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-23T18:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:09Z" name: model-chainer.18a90ef8bafcf86b namespace: kserve-ci-e2e-test resourceVersion: "17342" uid: b042c59e-3abc-4e9a-8e25-91cfa97bd327 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:25:14.888738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-02052-76989ff5b8-vh6wr namespace: kserve-ci-e2e-test resourceVersion: "23939" uid: 336f8215-db45-4321-996c-bfa21c9b9094 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-02052-76989ff5b8-vh6wr to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:25:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:25:14Z" name: sequence-graph-02052-76989ff5b8-vh6wr.18a90fa1a3ed0e10 namespace: kserve-ci-e2e-test resourceVersion: "23946" uid: 1c711248-0878-4889-a6b6-e1d64b13ebca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:25:15Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-02052-76989ff5b8-vh6wr namespace: kserve-ci-e2e-test resourceVersion: "23949" uid: 336f8215-db45-4321-996c-bfa21c9b9094 kind: Event lastTimestamp: "2026-04-23T18:25:15Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:25:15Z" name: sequence-graph-02052-76989ff5b8-vh6wr.18a90fa1bccc05b2 namespace: kserve-ci-e2e-test resourceVersion: "23961" uid: 13ffb89d-25a9-4c6f-99db-4cdc05118e04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-02052} kind: Pod name: sequence-graph-02052-76989ff5b8-vh6wr namespace: kserve-ci-e2e-test resourceVersion: "23941" uid: 336f8215-db45-4321-996c-bfa21c9b9094 kind: Event lastTimestamp: "2026-04-23T18:25:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:25:15Z" name: sequence-graph-02052-76989ff5b8-vh6wr.18a90fa1bdede8bc namespace: kserve-ci-e2e-test resourceVersion: "23963" uid: 4c474964-0f15-4e6d-90fc-8cb3bcdddc73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-02052} kind: Pod name: sequence-graph-02052-76989ff5b8-vh6wr namespace: kserve-ci-e2e-test resourceVersion: "23941" uid: 336f8215-db45-4321-996c-bfa21c9b9094 kind: Event lastTimestamp: "2026-04-23T18:25:15Z" message: 'Created container: sequence-graph-02052' metadata: creationTimestamp: "2026-04-23T18:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:25:15Z" name: sequence-graph-02052-76989ff5b8-vh6wr.18a90fa1c078568f namespace: kserve-ci-e2e-test resourceVersion: "23964" uid: 884bac69-c3f1-4d04-b028-c98542ebbb31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-02052} kind: Pod name: sequence-graph-02052-76989ff5b8-vh6wr namespace: kserve-ci-e2e-test resourceVersion: "23941" uid: 336f8215-db45-4321-996c-bfa21c9b9094 kind: Event lastTimestamp: "2026-04-23T18:25:15Z" message: Started container sequence-graph-02052 metadata: creationTimestamp: "2026-04-23T18:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:25:15Z" name: sequence-graph-02052-76989ff5b8-vh6wr.18a90fa1c0e788b6 namespace: kserve-ci-e2e-test resourceVersion: "23965" uid: 5ac81818-91df-4d5f-82f7-545262e0e82b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-02052} kind: Pod name: sequence-graph-02052-76989ff5b8-vh6wr namespace: kserve-ci-e2e-test resourceVersion: "23941" uid: 336f8215-db45-4321-996c-bfa21c9b9094 kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: Stopping container sequence-graph-02052 metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:29Z" name: sequence-graph-02052-76989ff5b8-vh6wr.18a91014cd66b8da namespace: kserve-ci-e2e-test resourceVersion: "27652" uid: d84ae51b-a5f3-4df5-80a4-e20f1257c855 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-02052} kind: Pod name: sequence-graph-02052-76989ff5b8-vh6wr namespace: kserve-ci-e2e-test resourceVersion: "23941" uid: 336f8215-db45-4321-996c-bfa21c9b9094 kind: Event lastTimestamp: "2026-04-23T18:33:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:57Z" name: sequence-graph-02052-76989ff5b8-vh6wr.18a910156204bfd1 namespace: kserve-ci-e2e-test resourceVersion: "28159" uid: 3867aa02-09f1-4a75-8f16-9a6a5c090b01 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:25:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-02052-76989ff5b8 namespace: kserve-ci-e2e-test resourceVersion: "23936" uid: 66e07015-aaa8-44bc-a93e-39d513ed652d kind: Event lastTimestamp: "2026-04-23T18:25:14Z" message: 'Created pod: sequence-graph-02052-76989ff5b8-vh6wr' metadata: creationTimestamp: "2026-04-23T18:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:14Z" name: sequence-graph-02052-76989ff5b8.18a90fa1a362cd35 namespace: kserve-ci-e2e-test resourceVersion: "23940" uid: 5dbe0e60-a0ec-4842-9ef2-be699922ebcc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:25:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-02052 namespace: kserve-ci-e2e-test resourceVersion: "23934" uid: 2be3567e-039d-468d-a4fb-5ccbf5ee1a6d kind: Event lastTimestamp: "2026-04-23T18:25:14Z" message: Scaled up replica set sequence-graph-02052-76989ff5b8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:14Z" name: sequence-graph-02052.18a90fa1a29c5593 namespace: kserve-ci-e2e-test resourceVersion: "23937" uid: cb4a64bb-0c07-4965-9a61-9d000266fbc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:25:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-02052 namespace: kserve-ci-e2e-test resourceVersion: "23951" uid: 8ec91846-17a8-43f2-86a5-97eed024596a kind: Event lastTimestamp: "2026-04-23T18:25:14Z" message: InferenceGraph [sequence-graph-02052] is Ready metadata: creationTimestamp: "2026-04-23T18:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:25:14Z" name: sequence-graph-02052.18a90fa1a631d988 namespace: kserve-ci-e2e-test resourceVersion: "23953" uid: cb22d8c0-8a1a-48aa-8715-6b1701322d8e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:25:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-02052 namespace: kserve-ci-e2e-test resourceVersion: "23935" uid: 706ff709-c362-44df-b3d7-19aa842ef736 kind: Event lastTimestamp: "2026-04-23T18:25:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:25:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:59Z" name: sequence-graph-02052.18a90fa52139ace7 namespace: kserve-ci-e2e-test resourceVersion: "24298" uid: b4704a41-0e31-4a92-8ccd-155d2719eafd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:25:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-02052 namespace: kserve-ci-e2e-test resourceVersion: "23935" uid: 706ff709-c362-44df-b3d7-19aa842ef736 kind: Event lastTimestamp: "2026-04-23T18:25:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:25:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:59Z" name: sequence-graph-02052.18a90fa521a72c88 namespace: kserve-ci-e2e-test resourceVersion: "24299" uid: 3aef662c-9a75-4f55-8a60-a85bc90256ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:14:29.716035Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-89c6b-5dcc6f7895-k8hq8 namespace: kserve-ci-e2e-test resourceVersion: "18242" uid: c1a7bd89-64d1-419a-9886-182284561c3a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-89c6b-5dcc6f7895-k8hq8 to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:14:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:14:29Z" name: sequence-graph-89c6b-5dcc6f7895-k8hq8.18a90f0b6ca2e382 namespace: kserve-ci-e2e-test resourceVersion: "18247" uid: abe4b0d8-2646-4b70-be1e-5abcb3d5f7cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-89c6b-5dcc6f7895-k8hq8 namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: c1a7bd89-64d1-419a-9886-182284561c3a kind: Event lastTimestamp: "2026-04-23T18:14:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-89c6b-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:29Z" name: sequence-graph-89c6b-5dcc6f7895-k8hq8.18a90f0b74188eda namespace: kserve-ci-e2e-test resourceVersion: "18256" uid: 6749bc00-2cba-4288-8fe0-f7cab6a412b6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-89c6b-5dcc6f7895-k8hq8 namespace: kserve-ci-e2e-test resourceVersion: "18250" uid: c1a7bd89-64d1-419a-9886-182284561c3a kind: Event lastTimestamp: "2026-04-23T18:14:30Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:14:30Z" name: sequence-graph-89c6b-5dcc6f7895-k8hq8.18a90f0ba91fca86 namespace: kserve-ci-e2e-test resourceVersion: "18268" uid: e01bb0dc-5d73-431d-b99e-7d5c593ac309 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89c6b} kind: Pod name: sequence-graph-89c6b-5dcc6f7895-k8hq8 namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: c1a7bd89-64d1-419a-9886-182284561c3a kind: Event lastTimestamp: "2026-04-23T18:14:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:30Z" name: sequence-graph-89c6b-5dcc6f7895-k8hq8.18a90f0baa53dd50 namespace: kserve-ci-e2e-test resourceVersion: "18271" uid: a1aa99fe-7df6-49e3-93dd-eedc9c2d1a93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89c6b} kind: Pod name: sequence-graph-89c6b-5dcc6f7895-k8hq8 namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: c1a7bd89-64d1-419a-9886-182284561c3a kind: Event lastTimestamp: "2026-04-23T18:14:30Z" message: 'Created container: sequence-graph-89c6b' metadata: creationTimestamp: "2026-04-23T18:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:30Z" name: sequence-graph-89c6b-5dcc6f7895-k8hq8.18a90f0bad6e0fbf namespace: kserve-ci-e2e-test resourceVersion: "18272" uid: b6043a02-0786-4592-9f4c-005f8e14f685 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89c6b} kind: Pod name: sequence-graph-89c6b-5dcc6f7895-k8hq8 namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: c1a7bd89-64d1-419a-9886-182284561c3a kind: Event lastTimestamp: "2026-04-23T18:14:30Z" message: Started container sequence-graph-89c6b metadata: creationTimestamp: "2026-04-23T18:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:14:30Z" name: sequence-graph-89c6b-5dcc6f7895-k8hq8.18a90f0badd93307 namespace: kserve-ci-e2e-test resourceVersion: "18273" uid: d206ef25-d9a8-4a85-84a6-fd46b91c0df1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89c6b} kind: Pod name: sequence-graph-89c6b-5dcc6f7895-k8hq8 namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: c1a7bd89-64d1-419a-9886-182284561c3a kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: Stopping container sequence-graph-89c6b metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:44Z" name: sequence-graph-89c6b-5dcc6f7895-k8hq8.18a90f7e99fcec02 namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: bd601c20-3627-44ff-9a2d-2f653e8cbbf5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:22:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89c6b} kind: Pod name: sequence-graph-89c6b-5dcc6f7895-k8hq8 namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: c1a7bd89-64d1-419a-9886-182284561c3a kind: Event lastTimestamp: "2026-04-23T18:23:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:12Z" name: sequence-graph-89c6b-5dcc6f7895-k8hq8.18a90f7f3ded1ce7 namespace: kserve-ci-e2e-test resourceVersion: "22382" uid: 5b478ed3-c266-41cd-86d5-21315ec3741b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-89c6b-5dcc6f7895 namespace: kserve-ci-e2e-test resourceVersion: "18239" uid: f1586a61-bff3-48ab-b4ed-83a37f001c7e kind: Event lastTimestamp: "2026-04-23T18:14:29Z" message: 'Created pod: sequence-graph-89c6b-5dcc6f7895-k8hq8' metadata: creationTimestamp: "2026-04-23T18:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:29Z" name: sequence-graph-89c6b-5dcc6f7895.18a90f0b6c1f4091 namespace: kserve-ci-e2e-test resourceVersion: "18245" uid: 0f68f61c-1d1e-4d97-b5ad-7ab88542006f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18237" uid: 12a2f582-05f2-470f-b5bb-545a872ce7f7 kind: Event lastTimestamp: "2026-04-23T18:14:29Z" message: Scaled up replica set sequence-graph-89c6b-5dcc6f7895 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:29Z" name: sequence-graph-89c6b.18a90f0b697dfb97 namespace: kserve-ci-e2e-test resourceVersion: "18240" uid: fafb2566-5154-4e80-b810-9ef91154405c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18253" uid: b3ec8feb-9055-4d09-960b-1b5806b870e0 kind: Event lastTimestamp: "2026-04-23T18:14:29Z" message: InferenceGraph [sequence-graph-89c6b] is Ready metadata: creationTimestamp: "2026-04-23T18:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:29Z" name: sequence-graph-89c6b.18a90f0b72eea9ad namespace: kserve-ci-e2e-test resourceVersion: "18255" uid: b7b69033-0697-4fd8-8e84-bc7c3b86457a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18230" uid: b3ec8feb-9055-4d09-960b-1b5806b870e0 kind: Event lastTimestamp: "2026-04-23T18:14:29Z" message: 'Failed to update status for InferenceGraph "sequence-graph-89c6b": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-89c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:29Z" name: sequence-graph-89c6b.18a90f0b7646c580 namespace: kserve-ci-e2e-test resourceVersion: "18258" uid: 08b3457e-9fb0-47c7-bc62-a81843859115 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18230" uid: b3ec8feb-9055-4d09-960b-1b5806b870e0 kind: Event lastTimestamp: "2026-04-23T18:14:29Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-89c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:29Z" name: sequence-graph-89c6b.18a90f0b76470b41 namespace: kserve-ci-e2e-test resourceVersion: "18260" uid: f18cb7ba-af3c-4646-a354-806d979e3cec reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18238" uid: 6043ad08-d508-4993-bda9-4794db52043e kind: Event lastTimestamp: "2026-04-23T18:14:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:44Z" name: sequence-graph-89c6b.18a90f0ee796a0a0 namespace: kserve-ci-e2e-test resourceVersion: "18378" uid: cc36f047-8f99-4855-8c0c-c87b23b986d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18238" uid: 6043ad08-d508-4993-bda9-4794db52043e kind: Event lastTimestamp: "2026-04-23T18:14:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:44Z" name: sequence-graph-89c6b.18a90f0ee8019e99 namespace: kserve-ci-e2e-test resourceVersion: "18380" uid: a29a89cc-c2e2-458f-8853-230aae28a13f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: 6043ad08-d508-4993-bda9-4794db52043e kind: Event lastTimestamp: "2026-04-23T18:14:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:59Z" name: sequence-graph-89c6b.18a90f1266a309c6 namespace: kserve-ci-e2e-test resourceVersion: "18484" uid: 2ca247aa-76f9-49ac-a813-3b424d1c1a22 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18379" uid: 6043ad08-d508-4993-bda9-4794db52043e kind: Event lastTimestamp: "2026-04-23T18:14:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:59Z" name: sequence-graph-89c6b.18a90f126711286e namespace: kserve-ci-e2e-test resourceVersion: "18486" uid: 541e95d2-06d1-41de-af51-9f3dab14b211 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:23:54.560919Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-89ff7-6f7fc5787f-7bn8w namespace: kserve-ci-e2e-test resourceVersion: "22764" uid: e5a67a37-ede8-4317-806e-a08a8901a495 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-89ff7-6f7fc5787f-7bn8w to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:23:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:23:54Z" name: sequence-graph-89ff7-6f7fc5787f-7bn8w.18a90f8ef003cf5f namespace: kserve-ci-e2e-test resourceVersion: "22769" uid: 770dac0d-1dcf-4973-9013-a1ba8a5e2ba3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:54Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-89ff7-6f7fc5787f-7bn8w namespace: kserve-ci-e2e-test resourceVersion: "22773" uid: e5a67a37-ede8-4317-806e-a08a8901a495 kind: Event lastTimestamp: "2026-04-23T18:23:54Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:23:54Z" name: sequence-graph-89ff7-6f7fc5787f-7bn8w.18a90f8f093683b1 namespace: kserve-ci-e2e-test resourceVersion: "22786" uid: 0c587bfc-ab44-496a-b725-9ee80df68d7b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89ff7} kind: Pod name: sequence-graph-89ff7-6f7fc5787f-7bn8w namespace: kserve-ci-e2e-test resourceVersion: "22767" uid: e5a67a37-ede8-4317-806e-a08a8901a495 kind: Event lastTimestamp: "2026-04-23T18:23:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:55Z" name: sequence-graph-89ff7-6f7fc5787f-7bn8w.18a90f8f0a68a718 namespace: kserve-ci-e2e-test resourceVersion: "22788" uid: 22522764-1bcf-4561-bc0a-1619517c805a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89ff7} kind: Pod name: sequence-graph-89ff7-6f7fc5787f-7bn8w namespace: kserve-ci-e2e-test resourceVersion: "22767" uid: e5a67a37-ede8-4317-806e-a08a8901a495 kind: Event lastTimestamp: "2026-04-23T18:23:55Z" message: 'Created container: sequence-graph-89ff7' metadata: creationTimestamp: "2026-04-23T18:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:55Z" name: sequence-graph-89ff7-6f7fc5787f-7bn8w.18a90f8f0d6d0e6d namespace: kserve-ci-e2e-test resourceVersion: "22789" uid: 29dc7e2e-52c6-4508-ac28-3fdedf9bbbd4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89ff7} kind: Pod name: sequence-graph-89ff7-6f7fc5787f-7bn8w namespace: kserve-ci-e2e-test resourceVersion: "22767" uid: e5a67a37-ede8-4317-806e-a08a8901a495 kind: Event lastTimestamp: "2026-04-23T18:23:55Z" message: Started container sequence-graph-89ff7 metadata: creationTimestamp: "2026-04-23T18:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:55Z" name: sequence-graph-89ff7-6f7fc5787f-7bn8w.18a90f8f0dda9920 namespace: kserve-ci-e2e-test resourceVersion: "22790" uid: 5a881af9-744f-4b09-a2e7-4c2a04eeb148 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89ff7} kind: Pod name: sequence-graph-89ff7-6f7fc5787f-7bn8w namespace: kserve-ci-e2e-test resourceVersion: "22767" uid: e5a67a37-ede8-4317-806e-a08a8901a495 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: Stopping container sequence-graph-89ff7 metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:04Z" name: sequence-graph-89ff7-6f7fc5787f-7bn8w.18a90f9148b8c758 namespace: kserve-ci-e2e-test resourceVersion: "22894" uid: 0f68eaf7-fa2e-462f-b498-d4c93ccf9fd4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:24:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-89ff7} kind: Pod name: sequence-graph-89ff7-6f7fc5787f-7bn8w namespace: kserve-ci-e2e-test resourceVersion: "22767" uid: e5a67a37-ede8-4317-806e-a08a8901a495 kind: Event lastTimestamp: "2026-04-23T18:24:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:31Z" name: sequence-graph-89ff7-6f7fc5787f-7bn8w.18a90f91c57539b4 namespace: kserve-ci-e2e-test resourceVersion: "23288" uid: ad1c4122-1d86-442f-a6d0-bdf3795c7b24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-89ff7-6f7fc5787f namespace: kserve-ci-e2e-test resourceVersion: "22761" uid: 78fd9296-87de-459b-958c-1013d3b0a6df kind: Event lastTimestamp: "2026-04-23T18:23:54Z" message: 'Created pod: sequence-graph-89ff7-6f7fc5787f-7bn8w' metadata: creationTimestamp: "2026-04-23T18:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:54Z" name: sequence-graph-89ff7-6f7fc5787f.18a90f8eef67c5e1 namespace: kserve-ci-e2e-test resourceVersion: "22765" uid: b5d2e0a6-6966-499b-92cf-0dc07edb6d36 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "22759" uid: 55d30e98-d21b-4433-a168-95344594ad32 kind: Event lastTimestamp: "2026-04-23T18:23:54Z" message: Scaled up replica set sequence-graph-89ff7-6f7fc5787f from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:54Z" name: sequence-graph-89ff7.18a90f8eeead5095 namespace: kserve-ci-e2e-test resourceVersion: "22762" uid: 1e691d50-45c5-42ad-a26e-5a0b0eb03811 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 06b3b4b0-9348-469a-a3fe-636522788f40 kind: Event lastTimestamp: "2026-04-23T18:23:54Z" message: InferenceGraph [sequence-graph-89ff7] is Ready metadata: creationTimestamp: "2026-04-23T18:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:54Z" name: sequence-graph-89ff7.18a90f8ef326e5b6 namespace: kserve-ci-e2e-test resourceVersion: "22778" uid: 7d9ac00a-6c91-41b5-964d-d323c9047f69 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:33:24.061712Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-60f5a-6466846f6-wtc9r namespace: kserve-ci-e2e-test resourceVersion: "27568" uid: 76f98d5f-9515-49f1-aba2-10550e02e50c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-60f5a-6466846f6-wtc9r to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:33:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:33:24Z" name: splitter-graph-60f5a-6466846f6-wtc9r.18a9101388e84f24 namespace: kserve-ci-e2e-test resourceVersion: "27573" uid: 1c251b07-fb81-4802-ba09-eb0b2b8b406a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:24Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-60f5a-6466846f6-wtc9r namespace: kserve-ci-e2e-test resourceVersion: "27571" uid: 76f98d5f-9515-49f1-aba2-10550e02e50c kind: Event lastTimestamp: "2026-04-23T18:33:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-60f5a-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:24Z" name: splitter-graph-60f5a-6466846f6-wtc9r.18a9101399423265 namespace: kserve-ci-e2e-test resourceVersion: "27590" uid: a09efffd-5018-4ce7-8307-ab47cff79746 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:25Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-60f5a-6466846f6-wtc9r namespace: kserve-ci-e2e-test resourceVersion: "27576" uid: 76f98d5f-9515-49f1-aba2-10550e02e50c kind: Event lastTimestamp: "2026-04-23T18:33:25Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:33:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:33:25Z" name: splitter-graph-60f5a-6466846f6-wtc9r.18a91013c5d58b56 namespace: kserve-ci-e2e-test resourceVersion: "27598" uid: f481f54e-5ac1-4ceb-bd2d-7f59628f54fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-60f5a} kind: Pod name: splitter-graph-60f5a-6466846f6-wtc9r namespace: kserve-ci-e2e-test resourceVersion: "27571" uid: 76f98d5f-9515-49f1-aba2-10550e02e50c kind: Event lastTimestamp: "2026-04-23T18:33:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:33:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:25Z" name: splitter-graph-60f5a-6466846f6-wtc9r.18a91013c74426ee namespace: kserve-ci-e2e-test resourceVersion: "27600" uid: 9fe55225-1eaf-4ab4-876a-a3e33f8b5375 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-60f5a} kind: Pod name: splitter-graph-60f5a-6466846f6-wtc9r namespace: kserve-ci-e2e-test resourceVersion: "27571" uid: 76f98d5f-9515-49f1-aba2-10550e02e50c kind: Event lastTimestamp: "2026-04-23T18:33:25Z" message: 'Created container: splitter-graph-60f5a' metadata: creationTimestamp: "2026-04-23T18:33:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:25Z" name: splitter-graph-60f5a-6466846f6-wtc9r.18a91013ca1342a5 namespace: kserve-ci-e2e-test resourceVersion: "27601" uid: d7db525e-eb6a-4dde-b362-1eefc5158433 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-60f5a} kind: Pod name: splitter-graph-60f5a-6466846f6-wtc9r namespace: kserve-ci-e2e-test resourceVersion: "27571" uid: 76f98d5f-9515-49f1-aba2-10550e02e50c kind: Event lastTimestamp: "2026-04-23T18:33:25Z" message: Started container splitter-graph-60f5a metadata: creationTimestamp: "2026-04-23T18:33:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:25Z" name: splitter-graph-60f5a-6466846f6-wtc9r.18a91013ca83b11c namespace: kserve-ci-e2e-test resourceVersion: "27602" uid: e51ede13-3dc7-41fb-b91c-e2a1f7ac02a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-60f5a} kind: Pod name: splitter-graph-60f5a-6466846f6-wtc9r namespace: kserve-ci-e2e-test resourceVersion: "27571" uid: 76f98d5f-9515-49f1-aba2-10550e02e50c kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Stopping container splitter-graph-60f5a metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: splitter-graph-60f5a-6466846f6-wtc9r.18a91016d0fbaee8 namespace: kserve-ci-e2e-test resourceVersion: "27871" uid: f8b88d2a-35c8-4f01-addd-7c218f59d491 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:33:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-60f5a} kind: Pod name: splitter-graph-60f5a-6466846f6-wtc9r namespace: kserve-ci-e2e-test resourceVersion: "27571" uid: 76f98d5f-9515-49f1-aba2-10550e02e50c kind: Event lastTimestamp: "2026-04-23T18:34:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:06Z" name: splitter-graph-60f5a-6466846f6-wtc9r.18a91017982bfa2d namespace: kserve-ci-e2e-test resourceVersion: "28236" uid: 313c06c5-7d52-4ee2-ae34-3352a84217ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-60f5a-6466846f6 namespace: kserve-ci-e2e-test resourceVersion: "27565" uid: 8fa41358-ada6-45d8-bdc2-b711a8949137 kind: Event lastTimestamp: "2026-04-23T18:33:24Z" message: 'Created pod: splitter-graph-60f5a-6466846f6-wtc9r' metadata: creationTimestamp: "2026-04-23T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:24Z" name: splitter-graph-60f5a-6466846f6.18a91013887fe9b0 namespace: kserve-ci-e2e-test resourceVersion: "27570" uid: 1af038fe-03d1-43ce-8a02-3fde7ef776a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-60f5a namespace: kserve-ci-e2e-test resourceVersion: "27563" uid: 4eb959ca-97b8-44d7-b578-f870ff5bba7a kind: Event lastTimestamp: "2026-04-23T18:33:24Z" message: Scaled up replica set splitter-graph-60f5a-6466846f6 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:24Z" name: splitter-graph-60f5a.18a91013877820a3 namespace: kserve-ci-e2e-test resourceVersion: "27566" uid: 2db098cc-93fd-4c00-a558-6dfa502fd187 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-60f5a namespace: kserve-ci-e2e-test resourceVersion: "27579" uid: 5ef240a1-61d1-46cd-946b-6cd748ec14c9 kind: Event lastTimestamp: "2026-04-23T18:33:24Z" message: InferenceGraph [splitter-graph-60f5a] is Ready metadata: creationTimestamp: "2026-04-23T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:24Z" name: splitter-graph-60f5a.18a910138c26b758 namespace: kserve-ci-e2e-test resourceVersion: "27581" uid: ea4bf37f-ed13-4833-84e1-96222793beb0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-60f5a namespace: kserve-ci-e2e-test resourceVersion: "27554" uid: 5ef240a1-61d1-46cd-946b-6cd748ec14c9 kind: Event lastTimestamp: "2026-04-23T18:33:24Z" message: 'Failed to update status for InferenceGraph "splitter-graph-60f5a": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-60f5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:24Z" name: splitter-graph-60f5a.18a910138e45887a namespace: kserve-ci-e2e-test resourceVersion: "27584" uid: 1ce39559-cc6e-48df-a26c-90fc1c2512a3 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-60f5a namespace: kserve-ci-e2e-test resourceVersion: "27554" uid: 5ef240a1-61d1-46cd-946b-6cd748ec14c9 kind: Event lastTimestamp: "2026-04-23T18:33:24Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-60f5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:24Z" name: splitter-graph-60f5a.18a910138e45cc60 namespace: kserve-ci-e2e-test resourceVersion: "27585" uid: 24d05bc5-ef88-451f-b3a1-fc396110bd17 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:34:38.386325Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-760ed-85f67b648f-2djgr namespace: kserve-ci-e2e-test resourceVersion: "28536" uid: 09342f26-fdec-4e8b-9720-f5252a85e1a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-760ed-85f67b648f-2djgr to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:34:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:34:38Z" name: splitter-graph-760ed-85f67b648f-2djgr.18a91024d6ffde3a namespace: kserve-ci-e2e-test resourceVersion: "28541" uid: d1047efa-c6fb-40c8-842b-45285e8223c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:38Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-760ed-85f67b648f-2djgr namespace: kserve-ci-e2e-test resourceVersion: "28538" uid: 09342f26-fdec-4e8b-9720-f5252a85e1a3 kind: Event lastTimestamp: "2026-04-23T18:34:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-760ed-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:38Z" name: splitter-graph-760ed-85f67b648f-2djgr.18a91024e1595c71 namespace: kserve-ci-e2e-test resourceVersion: "28557" uid: edb01290-a2ca-42bd-8bc8-2f8025f4a2a0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-760ed-85f67b648f-2djgr namespace: kserve-ci-e2e-test resourceVersion: "28544" uid: 09342f26-fdec-4e8b-9720-f5252a85e1a3 kind: Event lastTimestamp: "2026-04-23T18:34:39Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:34:39Z" name: splitter-graph-760ed-85f67b648f-2djgr.18a910251410f464 namespace: kserve-ci-e2e-test resourceVersion: "28564" uid: 1ac22556-3bb1-4c51-aee1-8b776b9f8282 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-760ed} kind: Pod name: splitter-graph-760ed-85f67b648f-2djgr namespace: kserve-ci-e2e-test resourceVersion: "28538" uid: 09342f26-fdec-4e8b-9720-f5252a85e1a3 kind: Event lastTimestamp: "2026-04-23T18:34:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:39Z" name: splitter-graph-760ed-85f67b648f-2djgr.18a910251546563b namespace: kserve-ci-e2e-test resourceVersion: "28566" uid: 589bb4c3-ece2-430f-8262-765c6c02b89f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-760ed} kind: Pod name: splitter-graph-760ed-85f67b648f-2djgr namespace: kserve-ci-e2e-test resourceVersion: "28538" uid: 09342f26-fdec-4e8b-9720-f5252a85e1a3 kind: Event lastTimestamp: "2026-04-23T18:34:39Z" message: 'Created container: splitter-graph-760ed' metadata: creationTimestamp: "2026-04-23T18:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:39Z" name: splitter-graph-760ed-85f67b648f-2djgr.18a91025180593ce namespace: kserve-ci-e2e-test resourceVersion: "28569" uid: 80f20066-5140-4656-92d6-da72f0d83cce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-760ed} kind: Pod name: splitter-graph-760ed-85f67b648f-2djgr namespace: kserve-ci-e2e-test resourceVersion: "28538" uid: 09342f26-fdec-4e8b-9720-f5252a85e1a3 kind: Event lastTimestamp: "2026-04-23T18:34:39Z" message: Started container splitter-graph-760ed metadata: creationTimestamp: "2026-04-23T18:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:39Z" name: splitter-graph-760ed-85f67b648f-2djgr.18a910251874fddb namespace: kserve-ci-e2e-test resourceVersion: "28570" uid: 4e980165-5413-44b6-a70d-9e6a089ca6fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-760ed} kind: Pod name: splitter-graph-760ed-85f67b648f-2djgr namespace: kserve-ci-e2e-test resourceVersion: "28538" uid: 09342f26-fdec-4e8b-9720-f5252a85e1a3 kind: Event lastTimestamp: "2026-04-23T18:42:53Z" message: Stopping container splitter-graph-760ed metadata: creationTimestamp: "2026-04-23T18:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:42:53Z" name: splitter-graph-760ed-85f67b648f-2djgr.18a9109806b62633 namespace: kserve-ci-e2e-test resourceVersion: "32011" uid: ceedceb3-b5d8-4638-83db-45a48b7f3b85 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-760ed} kind: Pod name: splitter-graph-760ed-85f67b648f-2djgr namespace: kserve-ci-e2e-test resourceVersion: "28538" uid: 09342f26-fdec-4e8b-9720-f5252a85e1a3 kind: Event lastTimestamp: "2026-04-23T18:43:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:43:20Z" name: splitter-graph-760ed-85f67b648f-2djgr.18a91098a345313d namespace: kserve-ci-e2e-test resourceVersion: "32242" uid: 85b617fb-593b-4ffc-8b11-1f91339bd89b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-760ed-85f67b648f namespace: kserve-ci-e2e-test resourceVersion: "28533" uid: b31571d3-7830-44fe-824f-ac20910193c8 kind: Event lastTimestamp: "2026-04-23T18:34:38Z" message: 'Created pod: splitter-graph-760ed-85f67b648f-2djgr' metadata: creationTimestamp: "2026-04-23T18:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:38Z" name: splitter-graph-760ed-85f67b648f.18a91024d693f4e7 namespace: kserve-ci-e2e-test resourceVersion: "28537" uid: 6cce501b-81e5-4465-89c9-5487b857437c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-760ed namespace: kserve-ci-e2e-test resourceVersion: "28531" uid: 4a7bac69-9c9b-4c6d-8a66-ae199f70e9ab kind: Event lastTimestamp: "2026-04-23T18:34:38Z" message: Scaled up replica set splitter-graph-760ed-85f67b648f from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:38Z" name: splitter-graph-760ed.18a91024d5e68bfb namespace: kserve-ci-e2e-test resourceVersion: "28534" uid: ac120921-0efb-4b0f-a06d-2ec887036db2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-760ed namespace: kserve-ci-e2e-test resourceVersion: "28551" uid: 4a167434-7efb-4169-afb9-a253372919b2 kind: Event lastTimestamp: "2026-04-23T18:34:38Z" message: InferenceGraph [splitter-graph-760ed] is Ready metadata: creationTimestamp: "2026-04-23T18:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:38Z" name: splitter-graph-760ed.18a91024db3e3d18 namespace: kserve-ci-e2e-test resourceVersion: "28553" uid: b4d4a197-4e15-497b-ae1a-a01e45186e76 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-760ed namespace: kserve-ci-e2e-test resourceVersion: "28524" uid: 4a167434-7efb-4169-afb9-a253372919b2 kind: Event lastTimestamp: "2026-04-23T18:34:38Z" message: 'Failed to update status for InferenceGraph "splitter-graph-760ed": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-760ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:38Z" name: splitter-graph-760ed.18a91024dd271d58 namespace: kserve-ci-e2e-test resourceVersion: "28554" uid: 1563df40-47a5-4823-9b2d-5b55409464e4 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-760ed namespace: kserve-ci-e2e-test resourceVersion: "28524" uid: 4a167434-7efb-4169-afb9-a253372919b2 kind: Event lastTimestamp: "2026-04-23T18:34:38Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-760ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:38Z" name: splitter-graph-760ed.18a91024dd27615c namespace: kserve-ci-e2e-test resourceVersion: "28555" uid: 4549ad4b-efd3-4765-a5d6-fa68d080f460 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:34:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-760ed namespace: kserve-ci-e2e-test resourceVersion: "28532" uid: 27676928-b8b7-4d03-b11f-2b82f5f273e7 kind: Event lastTimestamp: "2026-04-23T18:35:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:35:08Z" name: splitter-graph-760ed.18a9102854466773 namespace: kserve-ci-e2e-test resourceVersion: "28828" uid: 0852b119-e1fc-4085-babe-d248a15d0fbc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:34:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-760ed namespace: kserve-ci-e2e-test resourceVersion: "28532" uid: 27676928-b8b7-4d03-b11f-2b82f5f273e7 kind: Event lastTimestamp: "2026-04-23T18:35:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:35:08Z" name: splitter-graph-760ed.18a91028549a4d06 namespace: kserve-ci-e2e-test resourceVersion: "28829" uid: 2e5f7d65-f5d8-4a99-a683-3c3fe225cd1f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:24:04.777073Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22932" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-02052-predictor-7f966b6f9b-7bm79 to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:24:04Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a90f9150f204f9 namespace: kserve-ci-e2e-test resourceVersion: "22943" uid: 5276a758-232b-456e-b02e-0f994d31aee2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22953" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:24:05Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a90f916b57457b namespace: kserve-ci-e2e-test resourceVersion: "22989" uid: 2f3246cb-dcac-4faa-b161-5325cb202b1a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a90f916ccbceed namespace: kserve-ci-e2e-test resourceVersion: "22992" uid: b2b24c89-6785-444d-8706-761ae7a874c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a90f917267474d namespace: kserve-ci-e2e-test resourceVersion: "22993" uid: 9b1baf3c-02d0-4cb7-8efc-a5415c318614 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a90f9172ff77d3 namespace: kserve-ci-e2e-test resourceVersion: "22994" uid: bd1b1eff-4038-453c-b8be-d68379918cf6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a90f91730fdc78 namespace: kserve-ci-e2e-test resourceVersion: "22996" uid: 2a49c611-06a9-4067-aa29-6ce501bede2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a90f917766817c namespace: kserve-ci-e2e-test resourceVersion: "23000" uid: 08a2c49b-555f-4219-971e-c023b2459e38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:24:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:05Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a90f9177e66bee namespace: kserve-ci-e2e-test resourceVersion: "23001" uid: 55f317e7-437d-481f-8dd6-3d67f0b4df07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:24:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:33:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:32Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a90f919e82955b namespace: kserve-ci-e2e-test resourceVersion: "27797" uid: 76cf8587-566c-4231-9416-d842d3d0eb93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:29Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a91014d8d56593 namespace: kserve-ci-e2e-test resourceVersion: "27710" uid: 4fca2843-5698-455b-9b1f-62917f2904d9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:29Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a91014d8d6b56f namespace: kserve-ci-e2e-test resourceVersion: "27712" uid: 74678b75-1955-418a-a569-ffd4740674cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: c39b169b-ef78-41d1-a7ac-de0e3599683c kind: Event lastTimestamp: "2026-04-23T18:33:32Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:32Z" name: success-200-isvc-02052-predictor-7f966b6f9b-7bm79.18a9101566fe9ca9 namespace: kserve-ci-e2e-test resourceVersion: "27796" uid: 71eacd51-fa76-4d44-b3b2-4a2957feb579 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-02052-predictor-7f966b6f9b namespace: kserve-ci-e2e-test resourceVersion: "22922" uid: d68745d6-b552-4df4-8953-e265cf517ede kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: 'Created pod: success-200-isvc-02052-predictor-7f966b6f9b-7bm79' metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:04Z" name: success-200-isvc-02052-predictor-7f966b6f9b.18a90f915066b5f3 namespace: kserve-ci-e2e-test resourceVersion: "22934" uid: 5db34726-01d0-4572-b6b4-8738d943c565 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "22918" uid: f5aa74fa-607b-4357-904d-33f0d82e4d08 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: Scaled up replica set success-200-isvc-02052-predictor-7f966b6f9b from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:04Z" name: success-200-isvc-02052-predictor.18a90f914ebecb14 namespace: kserve-ci-e2e-test resourceVersion: "22925" uid: a91a1089-b2e1-4205-9539-572280a41c16 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:24:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "22920" uid: 86deb1b7-5809-4e77-8f48-3b998b0c9672 kind: Event lastTimestamp: "2026-04-23T18:24:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:34Z" name: success-200-isvc-02052-predictor.18a90f94cd63a8aa namespace: kserve-ci-e2e-test resourceVersion: "23316" uid: 6778cea1-c85d-47df-9f66-fc615e4bb2aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:24:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "22920" uid: 86deb1b7-5809-4e77-8f48-3b998b0c9672 kind: Event lastTimestamp: "2026-04-23T18:24:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:24:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:34Z" name: success-200-isvc-02052-predictor.18a90f94cdcbdd05 namespace: kserve-ci-e2e-test resourceVersion: "23317" uid: 5bf9ba1c-7630-461e-a62d-4cd5a48b6ca5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:24:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "23190" uid: 86deb1b7-5809-4e77-8f48-3b998b0c9672 kind: Event lastTimestamp: "2026-04-23T18:25:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:34Z" name: success-200-isvc-02052-predictor.18a90f9bcbd4eeee namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 3964e8aa-2257-4011-958d-34b1e653c0be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:24:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02052-predictor namespace: kserve-ci-e2e-test resourceVersion: "23190" uid: 86deb1b7-5809-4e77-8f48-3b998b0c9672 kind: Event lastTimestamp: "2026-04-23T18:25:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:25:34Z" name: success-200-isvc-02052-predictor.18a90f9bcc40913d namespace: kserve-ci-e2e-test resourceVersion: "24121" uid: 0310fefc-bab6-47cf-93d0-81f51808ddb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02052 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: 4c290fe4-0922-4dea-bee4-7b40a7d97995 kind: Event lastTimestamp: "2026-04-23T18:25:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:25:02Z" name: success-200-isvc-02052.18a90f914e467261 namespace: kserve-ci-e2e-test resourceVersion: "23831" uid: de515eaa-e425-40a7-9668-5d4983cac955 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02052 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: 4c290fe4-0922-4dea-bee4-7b40a7d97995 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-02052": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-02052": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:04Z" name: success-200-isvc-02052.18a90f914ec0d936 namespace: kserve-ci-e2e-test resourceVersion: "22924" uid: 2b5020ac-b7a4-4a05-8888-59664c00ab5f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02052 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: 4c290fe4-0922-4dea-bee4-7b40a7d97995 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-02052": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:24:04Z" name: success-200-isvc-02052.18a90f914ec17b6f namespace: kserve-ci-e2e-test resourceVersion: "22928" uid: cdeaf07f-f29e-4d5a-8933-de414589c2f5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:25:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02052 namespace: kserve-ci-e2e-test resourceVersion: "23828" uid: 4c290fe4-0922-4dea-bee4-7b40a7d97995 kind: Event lastTimestamp: "2026-04-23T18:25:02Z" message: InferenceService [success-200-isvc-02052] is Ready metadata: creationTimestamp: "2026-04-23T18:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:25:02Z" name: success-200-isvc-02052.18a90f9ead4cf767 namespace: kserve-ci-e2e-test resourceVersion: "23830" uid: cfe3658b-089e-496a-8439-0b6b94ec38ee reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:12:33.727789Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16932" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-03167-predictor-79d9b874d8-7mgnd to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:12:33Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90ef06b32be09 namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: a4c19317-66e3-4fd8-b519-aa8f51799c83 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16950" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:12:34Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:12:34Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90ef0853d8d97 namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: d75dc2e8-ea44-4106-9e91-ecfd1f79e5f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:12:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:34Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90ef086d027d9 namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: b6d9f451-8315-4f40-8f6a-f330ab9ce2df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:12:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:34Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90ef089d3786a namespace: kserve-ci-e2e-test resourceVersion: "16974" uid: b6034ab0-60cc-4f6a-a2f2-6cbb6cfff9f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:12:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:34Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90ef08a4ba084 namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: f1ca4282-b490-440e-80e8-58e9b9e5e329 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:12:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:34Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90ef08a59895a namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 570fa220-2a1c-46a2-91d6-d94e81401e91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:12:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:34Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90ef08d270858 namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: 37a3eeb0-8b2f-4873-bacf-f8b62a24be55 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:12:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:34Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90ef08da713c7 namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: 1bcb5402-719b-41a2-9725-cdb858cc377b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:12:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:22:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:01Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90ef0dfdf71b8 namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 2ca66a3a-87b8-4483-b26b-f7e778261f7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:58Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90f73efe8c56d namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: 59997c8f-ec9e-482f-8fa6-92522d62b67b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:58Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90f73efe93224 namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 196b5bbb-75a1-45e0-bc69-78faace9bf4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 459bf2e3-dad1-46f5-bc0c-8705379a964c kind: Event lastTimestamp: "2026-04-23T18:22:01Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:01Z" name: success-200-isvc-03167-predictor-79d9b874d8-7mgnd.18a90f74a85827c7 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: b953ae5a-eeb8-4608-9ee7-a341657c9760 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-03167-predictor-79d9b874d8 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: e5409fe4-6179-4486-917f-225b5e5590e8 kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: 'Created pod: success-200-isvc-03167-predictor-79d9b874d8-7mgnd' metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:33Z" name: success-200-isvc-03167-predictor-79d9b874d8.18a90ef06aa7fca8 namespace: kserve-ci-e2e-test resourceVersion: "16935" uid: 109e255f-c8c5-4c22-b0b5-b19ae763a891 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: 379e76ba-4324-4a91-962c-4fd7b9183c2b kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: Scaled up replica set success-200-isvc-03167-predictor-79d9b874d8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:33Z" name: success-200-isvc-03167-predictor.18a90ef067449a85 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: d012dd50-1bb0-40a9-bbfb-edbb677560e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: 586d34f6-6ccd-46fd-ad77-356e3a216f17 kind: Event lastTimestamp: "2026-04-23T18:12:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:48Z" name: success-200-isvc-03167-predictor.18a90ef3e6364120 namespace: kserve-ci-e2e-test resourceVersion: "17152" uid: ed261211-a6d3-4320-8e5d-1a30cffbaf77 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: 586d34f6-6ccd-46fd-ad77-356e3a216f17 kind: Event lastTimestamp: "2026-04-23T18:12:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:48Z" name: success-200-isvc-03167-predictor.18a90ef3e6a9f3f4 namespace: kserve-ci-e2e-test resourceVersion: "17154" uid: 666a39d5-5616-4136-a770-8f9c641b8fbb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:13:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "17153" uid: 586d34f6-6ccd-46fd-ad77-356e3a216f17 kind: Event lastTimestamp: "2026-04-23T18:14:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:03Z" name: success-200-isvc-03167-predictor.18a90ef765668d6e namespace: kserve-ci-e2e-test resourceVersion: "18018" uid: 5112f358-6a18-4d4e-afdd-15f189cc7a13 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:13:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03167-predictor namespace: kserve-ci-e2e-test resourceVersion: "17153" uid: 586d34f6-6ccd-46fd-ad77-356e3a216f17 kind: Event lastTimestamp: "2026-04-23T18:14:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:03Z" name: success-200-isvc-03167-predictor.18a90ef765f19642 namespace: kserve-ci-e2e-test resourceVersion: "18019" uid: 87596510-416e-47a2-97e1-fa76b243ba02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-03167 namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: fa17c362-57cd-4c73-8395-79c8141a3714 kind: Event lastTimestamp: "2026-04-23T18:13:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:31Z" name: success-200-isvc-03167.18a90ef0670e337b namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: b9eff661-d200-4324-9deb-87b0b8d88fad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-03167 namespace: kserve-ci-e2e-test resourceVersion: "17718" uid: fa17c362-57cd-4c73-8395-79c8141a3714 kind: Event lastTimestamp: "2026-04-23T18:13:31Z" message: InferenceService [success-200-isvc-03167] is Ready metadata: creationTimestamp: "2026-04-23T18:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:31Z" name: success-200-isvc-03167.18a90efdee7a1c66 namespace: kserve-ci-e2e-test resourceVersion: "17720" uid: ff08e3eb-cef6-4407-8929-9a562e6b0f6c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:21:58.723723Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:21:58Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f73f7947a78 namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: da206283-35b5-4e1c-b551-fb1ae9281a1a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-5d865-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:58Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f73ffb39af2 namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: 7d47c144-432a-408f-8a46-2755ff46e728 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:21:59Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f7434756179 namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 1608531c-78db-4dd2-afcb-e1d70193253b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f7435e63005 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: 38df3939-7277-4092-b723-4e99389b0a33 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f7438b58958 namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: 44fe2563-b120-4587-912a-6bd5f54740f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f7439201156 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: 64a1eca2-c41a-45c0-b051-f9984025370f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f74392c1e62 namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: eca07cb7-6756-4ab4-85a7-62c3653e70ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f743bba4fa5 namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 00c8f4b9-ae2c-4cf4-a863-602f69f1a224 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:21:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:59Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f743c2a585e namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: 5fdac3cc-2273-473c-8c0c-c579c58e7160 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:22:37Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:37Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f74a38c45b8 namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: b045836d-322b-4208-9f0a-c65a7c430567 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:23:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:08Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f844f8602a5 namespace: kserve-ci-e2e-test resourceVersion: "22286" uid: 38bba93f-470f-4212-9681-0589c3f644cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:23:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:08Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f844f866834 namespace: kserve-ci-e2e-test resourceVersion: "22288" uid: 4f2dd3d2-5c07-4d4f-8558-bb82841e43ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f904c9ba-5277-42f1-8335-104599745997 kind: Event lastTimestamp: "2026-04-23T18:23:12Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:12Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f.18a90f852bd93679 namespace: kserve-ci-e2e-test resourceVersion: "22389" uid: baf1eaf3-5aed-46f4-9fe6-05d1e4525cd2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5d865-predictor-6cddf5c7f8 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 45e9e83a-e94d-4508-aae1-3828869d7e21 kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: 'Created pod: success-200-isvc-5d865-predictor-6cddf5c7f8-6dm2f' metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:21:58Z" name: success-200-isvc-5d865-predictor-6cddf5c7f8.18a90f73f7195543 namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: 13c8a94f-cb83-42b8-8924-41cd00afd3b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 12d8a37a-db4b-4dab-a868-5633543076b4 kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: Scaled up replica set success-200-isvc-5d865-predictor-6cddf5c7f8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:21:58Z" name: success-200-isvc-5d865-predictor.18a90f73f18908bc namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: c332b962-3ce1-47cb-bd0a-ccd0a5def816 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: fab356b6-b225-4144-9622-e4ca212c7905 kind: Event lastTimestamp: "2026-04-23T18:22:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:28Z" name: success-200-isvc-5d865-predictor.18a90f77700e8903 namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 4e950283-1428-4f01-a54c-561b536ab430 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: fab356b6-b225-4144-9622-e4ca212c7905 kind: Event lastTimestamp: "2026-04-23T18:22:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:28Z" name: success-200-isvc-5d865-predictor.18a90f77707fe5a0 namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: a8aca06a-4562-4790-aeb8-403330a0c81b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: fab356b6-b225-4144-9622-e4ca212c7905 kind: Event lastTimestamp: "2026-04-23T18:22:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:22:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:58Z" name: success-200-isvc-5d865-predictor.18a90f7e6df9f329 namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: b42c5e9a-3ec8-4285-a789-390376f851e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:22:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5d865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: fab356b6-b225-4144-9622-e4ca212c7905 kind: Event lastTimestamp: "2026-04-23T18:22:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:22:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:58Z" name: success-200-isvc-5d865-predictor.18a90f7e6e64b658 namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: 0fa9cb52-46fc-4c9c-904a-9a8649f03017 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 91e2ec91-0c5a-4c4f-a668-5f80ffbb8c79 kind: Event lastTimestamp: "2026-04-23T18:22:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:47Z" name: success-200-isvc-5d865.18a90f73f1518955 namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: 6461152d-8ee6-4e6d-88f0-a46b7cfc0d9e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 91e2ec91-0c5a-4c4f-a668-5f80ffbb8c79 kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: 'Failed to update status for InferenceService "success-200-isvc-5d865": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5d865": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:21:58Z" name: success-200-isvc-5d865.18a90f73f204a24d namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: bc3c310d-5835-4774-bc66-14fe26cf8813 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 91e2ec91-0c5a-4c4f-a668-5f80ffbb8c79 kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5d865": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:21:58Z" name: success-200-isvc-5d865.18a90f73f205000a namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 89763c07-8e60-4c8d-b188-e65db9972d48 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 91e2ec91-0c5a-4c4f-a668-5f80ffbb8c79 kind: Event lastTimestamp: "2026-04-23T18:22:47Z" message: InferenceService [success-200-isvc-5d865] is Ready metadata: creationTimestamp: "2026-04-23T18:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:47Z" name: success-200-isvc-5d865.18a90f7f5e7ebaa9 namespace: kserve-ci-e2e-test resourceVersion: "21998" uid: 89c4f308-87b1-4dde-8af4-73bc29c99b5b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:32:24.029177Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:32:24Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a910058eb092ee namespace: kserve-ci-e2e-test resourceVersion: "27015" uid: f6b89457-ea42-4794-a2a9-5f9e8e61439e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-60f5a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:24Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a910059a6b3344 namespace: kserve-ci-e2e-test resourceVersion: "27047" uid: c8a88eab-5137-4fbe-8445-d42639b1bbdc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:32:25Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:32:25Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91005cbe7295b namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 89aa94b9-8464-4792-ade3-e50fb5c6d00e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:32:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:25Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91005cd7143a8 namespace: kserve-ci-e2e-test resourceVersion: "27072" uid: 8ee2d624-fa0b-47fa-b191-c96dc9f03ae5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:32:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:25Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91005d0dcf6a3 namespace: kserve-ci-e2e-test resourceVersion: "27074" uid: 0af88ab3-4c13-4a15-ba84-60d24a73f64f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:32:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:25Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91005d1526b73 namespace: kserve-ci-e2e-test resourceVersion: "27075" uid: 575450de-8bae-4318-8889-21dd07707136 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:32:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:25Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91005d15efb34 namespace: kserve-ci-e2e-test resourceVersion: "27076" uid: 605f594c-b069-4961-b8b9-a2711922c4c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:32:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:25Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91005d48b96e7 namespace: kserve-ci-e2e-test resourceVersion: "27077" uid: d2bdf727-868b-457b-b8f9-6af1ffc95da6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:32:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:25Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91005d5044a36 namespace: kserve-ci-e2e-test resourceVersion: "27078" uid: a35e0df5-1475-406c-8ada-3e1388dddcdf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:33:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:41Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91005f90f47d1 namespace: kserve-ci-e2e-test resourceVersion: "28017" uid: 16a59bff-0d83-4a80-aa23-963391aa1f34 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91016d6e96b0f namespace: kserve-ci-e2e-test resourceVersion: "27906" uid: bcbcc2c2-c7ac-48f7-b77b-fc804603a576 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91016d6ea2135 namespace: kserve-ci-e2e-test resourceVersion: "27909" uid: 35bc36e0-a01d-4dff-b5d6-2008a73ba79b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: a4ae74b8-bf02-4534-b6aa-1a00bcd795a4 kind: Event lastTimestamp: "2026-04-23T18:33:41Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp 10.133.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:41Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr.18a91017ab376d7e namespace: kserve-ci-e2e-test resourceVersion: "28016" uid: ed94a063-f88d-4bd7-acda-3ae8e48bee80 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-60f5a-predictor-6cbdd87d54 namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: eef48bc0-68a6-4721-b464-6e67d1e17368 kind: Event lastTimestamp: "2026-04-23T18:32:24Z" message: 'Created pod: success-200-isvc-60f5a-predictor-6cbdd87d54-zcvbr' metadata: creationTimestamp: "2026-04-23T18:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:32:24Z" name: success-200-isvc-60f5a-predictor-6cbdd87d54.18a910058e10371f namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: 86e915d5-263c-4753-a574-a587861003d5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26978" uid: 47cec557-b8f7-427b-8029-315e0adb0897 kind: Event lastTimestamp: "2026-04-23T18:32:23Z" message: Scaled up replica set success-200-isvc-60f5a-predictor-6cbdd87d54 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:32:23Z" name: success-200-isvc-60f5a-predictor.18a9100589410b01 namespace: kserve-ci-e2e-test resourceVersion: "26993" uid: 61f6c48a-b9f5-4dd6-ac0a-a13dc43bf3cd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: 597f297d-0b40-4e69-aeee-bd81ec8bb5b8 kind: Event lastTimestamp: "2026-04-23T18:32:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:32:38Z" name: success-200-isvc-60f5a-predictor.18a9100907a50a09 namespace: kserve-ci-e2e-test resourceVersion: "27198" uid: 66d9038b-18e5-429c-8719-4b817cbd0d48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: 597f297d-0b40-4e69-aeee-bd81ec8bb5b8 kind: Event lastTimestamp: "2026-04-23T18:32:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:32:38Z" name: success-200-isvc-60f5a-predictor.18a9100908098dc0 namespace: kserve-ci-e2e-test resourceVersion: "27200" uid: 1429b540-8909-4920-8d51-43a0d4ccd41b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:32:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27199" uid: 597f297d-0b40-4e69-aeee-bd81ec8bb5b8 kind: Event lastTimestamp: "2026-04-23T18:33:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:23Z" name: success-200-isvc-60f5a-predictor.18a9100c86b87173 namespace: kserve-ci-e2e-test resourceVersion: "27555" uid: e7ccd712-c14f-4865-b0f1-d260d73ce505 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:32:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60f5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27199" uid: 597f297d-0b40-4e69-aeee-bd81ec8bb5b8 kind: Event lastTimestamp: "2026-04-23T18:33:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:23Z" name: success-200-isvc-60f5a-predictor.18a9100c872b0847 namespace: kserve-ci-e2e-test resourceVersion: "27556" uid: cb88ba50-a3c9-4da7-9b04-953328c6e912 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60f5a namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 685e1b60-4472-4898-b73b-7f6758188272 kind: Event lastTimestamp: "2026-04-23T18:33:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:11Z" name: success-200-isvc-60f5a.18a9100588f522ba namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: 5fc416a8-6374-46ef-a8f0-b39fb2b28e68 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60f5a namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 685e1b60-4472-4898-b73b-7f6758188272 kind: Event lastTimestamp: "2026-04-23T18:32:23Z" message: 'Failed to update status for InferenceService "success-200-isvc-60f5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-60f5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:32:23Z" name: success-200-isvc-60f5a.18a9100589eb905f namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: bcf91752-5c1e-47b1-af67-ff0b14f23b16 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60f5a namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 685e1b60-4472-4898-b73b-7f6758188272 kind: Event lastTimestamp: "2026-04-23T18:32:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-60f5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:32:23Z" name: success-200-isvc-60f5a.18a9100589ebd894 namespace: kserve-ci-e2e-test resourceVersion: "26999" uid: e6aa2930-205a-4227-8178-754d1992cb42 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60f5a namespace: kserve-ci-e2e-test resourceVersion: "27461" uid: 685e1b60-4472-4898-b73b-7f6758188272 kind: Event lastTimestamp: "2026-04-23T18:33:11Z" message: InferenceService [success-200-isvc-60f5a] is Ready metadata: creationTimestamp: "2026-04-23T18:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:11Z" name: success-200-isvc-60f5a.18a91010b41f1e95 namespace: kserve-ci-e2e-test resourceVersion: "27462" uid: 480de407-2927-423b-ad6f-201df277b71e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:33:38.275726Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27908" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-760ed-predictor-5856fb9f74-2rclx to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91016d820e891 namespace: kserve-ci-e2e-test resourceVersion: "27916" uid: 27a50758-e664-4af1-bb23-f969a4528cc1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27926" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91016f20425db namespace: kserve-ci-e2e-test resourceVersion: "27967" uid: fd10a6f8-09cd-4f27-b4b5-1d73ff29d1fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91016f35cecf3 namespace: kserve-ci-e2e-test resourceVersion: "27969" uid: 8cf3d581-037e-4244-a60e-e05f9fe82ff7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91016f72b1ca6 namespace: kserve-ci-e2e-test resourceVersion: "27971" uid: bb582e33-e5e2-409f-a5fd-27327504f914 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91016f7a5acda namespace: kserve-ci-e2e-test resourceVersion: "27972" uid: 59cfe457-6663-498e-a3a3-04f2e1f5f53a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91016f7b511f2 namespace: kserve-ci-e2e-test resourceVersion: "27973" uid: d9ea9521-bf06-4a62-9578-1f84f3984695 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91016fb81c6c0 namespace: kserve-ci-e2e-test resourceVersion: "27977" uid: 0f28d3b5-bde0-42a5-bca1-c08f6b8ec724 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91016fc152055 namespace: kserve-ci-e2e-test resourceVersion: "27978" uid: 0503c645-5dff-49af-95c1-684421e72643 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:33:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:42:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:33:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:42:56Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91017457653d3 namespace: kserve-ci-e2e-test resourceVersion: "32065" uid: 0b9a2614-4472-4539-9ea9-08c01381af0a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:42:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:42:53Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a910980bd53fa8 namespace: kserve-ci-e2e-test resourceVersion: "32026" uid: d9c30254-1a14-4be2-bf18-2f634f92fe55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:42:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:42:53Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a910980bd5c54f namespace: kserve-ci-e2e-test resourceVersion: "32031" uid: 01f34d64-f257-4bcf-ad5a-69ab6438a951 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:42:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 97047322-b35c-453c-858d-7f2bbb7c8806 kind: Event lastTimestamp: "2026-04-23T18:42:56Z" message: 'Readiness probe failed: Get "https://10.133.0.46:8643/healthz": dial tcp 10.133.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:42:56Z" name: success-200-isvc-760ed-predictor-5856fb9f74-2rclx.18a91098b9ca3e61 namespace: kserve-ci-e2e-test resourceVersion: "32064" uid: bbdc8b7b-ddf4-47f0-8c97-b8b5e4ff920f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-760ed-predictor-5856fb9f74 namespace: kserve-ci-e2e-test resourceVersion: "27892" uid: 39e2a2fa-a0a6-4e02-8176-5924b09aaf10 kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'Created pod: success-200-isvc-760ed-predictor-5856fb9f74-2rclx' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor-5856fb9f74.18a91016d7914625 namespace: kserve-ci-e2e-test resourceVersion: "27911" uid: 7e095e4e-7867-4742-aeff-a25824023566 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "27884" uid: 1c0b2fda-166a-4d8a-8b62-4f3c81458b3f kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: Scaled up replica set success-200-isvc-760ed-predictor-5856fb9f74 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed-predictor.18a91016d5a58d87 namespace: kserve-ci-e2e-test resourceVersion: "27898" uid: 8404879b-547a-4722-9ad4-50c6aedfbf41 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:33:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "27890" uid: af236faa-abba-43cb-bbd2-1db2b8446328 kind: Event lastTimestamp: "2026-04-23T18:34:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:08Z" name: success-200-isvc-760ed-predictor.18a9101a545ac8d4 namespace: kserve-ci-e2e-test resourceVersion: "28247" uid: 434d9dd7-1851-453f-85bb-03b2ef4d6ec7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:33:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "27890" uid: af236faa-abba-43cb-bbd2-1db2b8446328 kind: Event lastTimestamp: "2026-04-23T18:34:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:08Z" name: success-200-isvc-760ed-predictor.18a9101a54be9f0f namespace: kserve-ci-e2e-test resourceVersion: "28248" uid: 2cd42f6d-b270-4d03-b126-16fc33e72951 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:34:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "28123" uid: af236faa-abba-43cb-bbd2-1db2b8446328 kind: Event lastTimestamp: "2026-04-23T18:34:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:38Z" name: success-200-isvc-760ed-predictor.18a9102152d5b6cb namespace: kserve-ci-e2e-test resourceVersion: "28522" uid: b14125a1-c15e-4ab3-ab67-883a75e49a96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:34:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-760ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "28123" uid: af236faa-abba-43cb-bbd2-1db2b8446328 kind: Event lastTimestamp: "2026-04-23T18:34:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:38Z" name: success-200-isvc-760ed-predictor.18a910215335b8f5 namespace: kserve-ci-e2e-test resourceVersion: "28523" uid: 0bb9cc12-2c3d-4d92-90f1-2257b7f891fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-760ed namespace: kserve-ci-e2e-test resourceVersion: "27873" uid: 63516abd-f5c7-4f18-9a6d-13ad8afd77ac kind: Event lastTimestamp: "2026-04-23T18:34:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:26Z" name: success-200-isvc-760ed.18a91016d5519a74 namespace: kserve-ci-e2e-test resourceVersion: "28409" uid: c249eebf-9b22-49ed-9841-b8eec85887b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-760ed namespace: kserve-ci-e2e-test resourceVersion: "27873" uid: 63516abd-f5c7-4f18-9a6d-13ad8afd77ac kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'Failed to update status for InferenceService "success-200-isvc-760ed": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-760ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed.18a91016d5fae0c4 namespace: kserve-ci-e2e-test resourceVersion: "27900" uid: 4fca46cb-be15-43e4-9ff5-08d59f959ca2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-760ed namespace: kserve-ci-e2e-test resourceVersion: "27873" uid: 63516abd-f5c7-4f18-9a6d-13ad8afd77ac kind: Event lastTimestamp: "2026-04-23T18:33:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-760ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:33:38Z" name: success-200-isvc-760ed.18a91016d5fb4250 namespace: kserve-ci-e2e-test resourceVersion: "27902" uid: fb14b61e-779c-4681-8a10-377074cc522c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-760ed namespace: kserve-ci-e2e-test resourceVersion: "28407" uid: 63516abd-f5c7-4f18-9a6d-13ad8afd77ac kind: Event lastTimestamp: "2026-04-23T18:34:26Z" message: InferenceService [success-200-isvc-760ed] is Ready metadata: creationTimestamp: "2026-04-23T18:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:26Z" name: success-200-isvc-760ed.18a91022002f2957 namespace: kserve-ci-e2e-test resourceVersion: "28408" uid: 1663d174-7e95-4627-b95e-33867484c678 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:23:08.942535Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22289" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-83978-predictor-6788576868-g8bz9 to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:23:08Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a90f8450f28dc4 namespace: kserve-ci-e2e-test resourceVersion: "22293" uid: 14349157-7706-4f81-b4db-ba7aa808b8ef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22297" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:23:09Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a90f846aa01c6a namespace: kserve-ci-e2e-test resourceVersion: "22332" uid: bbb00563-508b-4222-acd6-28cb62267082 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a90f846c02fe56 namespace: kserve-ci-e2e-test resourceVersion: "22334" uid: 58cc48ff-df92-4881-9c26-ffa6057ba737 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a90f846ff0a698 namespace: kserve-ci-e2e-test resourceVersion: "22336" uid: a1db48a4-ef3b-4b0c-958f-36fc68a8663c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a90f84709b8ddf namespace: kserve-ci-e2e-test resourceVersion: "22338" uid: 7df86767-9c4e-48b2-aee9-14e5d101ae4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a90f8470a9eca2 namespace: kserve-ci-e2e-test resourceVersion: "22339" uid: 43ffad75-4b93-40aa-8740-73dfc66f0bf1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a90f8474cb22a9 namespace: kserve-ci-e2e-test resourceVersion: "22343" uid: 368beeb9-4dc2-4e16-9ba2-e7ca99cc37f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:23:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:09Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a90f84754648f4 namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: d89e2a3f-b7e2-433f-9371-e9be28b7ced5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:23:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:32:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:26Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a90f84c41dddfd namespace: kserve-ci-e2e-test resourceVersion: "27103" uid: a89fa746-a6e2-4626-87b7-4306c803b42d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:32:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:23Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a91005876f34f7 namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: f850e573-6582-41d0-8dad-e9517637ac45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:32:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:23Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a91005876fccb2 namespace: kserve-ci-e2e-test resourceVersion: "26980" uid: 02a025c4-05fd-45c9-8adc-e6fb124d3b63 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:32:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-83978-predictor-6788576868-g8bz9 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 80140f60-c462-4ad3-9313-f9bee99889bf kind: Event lastTimestamp: "2026-04-23T18:32:26Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:32:26Z" name: success-200-isvc-83978-predictor-6788576868-g8bz9.18a91006386ba8f6 namespace: kserve-ci-e2e-test resourceVersion: "27102" uid: 562ee2e9-aa37-4546-bda7-97962e0e16a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-83978-predictor-6788576868 namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: b2bdaf04-752b-45c0-a6ca-e631cfdfa213 kind: Event lastTimestamp: "2026-04-23T18:23:08Z" message: 'Created pod: success-200-isvc-83978-predictor-6788576868-g8bz9' metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:08Z" name: success-200-isvc-83978-predictor-6788576868.18a90f845063e13c namespace: kserve-ci-e2e-test resourceVersion: "22290" uid: 18308108-d38e-42fd-9450-3dc881827f0b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22272" uid: 77993b2b-5853-46d9-baa7-3803a3e420e7 kind: Event lastTimestamp: "2026-04-23T18:23:08Z" message: Scaled up replica set success-200-isvc-83978-predictor-6788576868 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:08Z" name: success-200-isvc-83978-predictor.18a90f844e9ede32 namespace: kserve-ci-e2e-test resourceVersion: "22280" uid: d65d82d5-ec41-4c73-9feb-502d81cb87ef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: a2932923-b83f-4786-932e-c003123f41c9 kind: Event lastTimestamp: "2026-04-23T18:23:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:38Z" name: success-200-isvc-83978-predictor.18a90f87cd96fdf6 namespace: kserve-ci-e2e-test resourceVersion: "22612" uid: e9b1665d-0c4b-4cc7-8c1e-2abc498ae22b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:23:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: a2932923-b83f-4786-932e-c003123f41c9 kind: Event lastTimestamp: "2026-04-23T18:23:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:23:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:38Z" name: success-200-isvc-83978-predictor.18a90f87ce0274c0 namespace: kserve-ci-e2e-test resourceVersion: "22613" uid: f41211c6-5607-4391-813a-4b06eca5ee2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:23:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22492" uid: a2932923-b83f-4786-932e-c003123f41c9 kind: Event lastTimestamp: "2026-04-23T18:24:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:23Z" name: success-200-isvc-83978-predictor.18a90f8ecc318eae namespace: kserve-ci-e2e-test resourceVersion: "23230" uid: c944c97a-e90d-4e83-b1b3-488cef059381 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:23:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-83978-predictor namespace: kserve-ci-e2e-test resourceVersion: "22492" uid: a2932923-b83f-4786-932e-c003123f41c9 kind: Event lastTimestamp: "2026-04-23T18:24:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:24:23Z" name: success-200-isvc-83978-predictor.18a90f8ecc889c3f namespace: kserve-ci-e2e-test resourceVersion: "23231" uid: c1012e18-7be0-41c3-8ca1-e5c9a50a93a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-83978 namespace: kserve-ci-e2e-test resourceVersion: "22246" uid: 8e3b5190-3bb5-4ba3-b569-9d7ab526367e kind: Event lastTimestamp: "2026-04-23T18:23:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:56Z" name: success-200-isvc-83978.18a90f844e6bc6d1 namespace: kserve-ci-e2e-test resourceVersion: "22817" uid: fa2d737c-d5c6-4c23-9eaa-83f94282677b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-83978 namespace: kserve-ci-e2e-test resourceVersion: "22246" uid: 8e3b5190-3bb5-4ba3-b569-9d7ab526367e kind: Event lastTimestamp: "2026-04-23T18:23:08Z" message: 'Failed to update status for InferenceService "success-200-isvc-83978": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-83978": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:08Z" name: success-200-isvc-83978.18a90f844eaea858 namespace: kserve-ci-e2e-test resourceVersion: "22282" uid: 56444e32-8db5-4046-a8f7-56bfe3544f51 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-83978 namespace: kserve-ci-e2e-test resourceVersion: "22246" uid: 8e3b5190-3bb5-4ba3-b569-9d7ab526367e kind: Event lastTimestamp: "2026-04-23T18:23:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-83978": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:08Z" name: success-200-isvc-83978.18a90f844eaeebdc namespace: kserve-ci-e2e-test resourceVersion: "22285" uid: 9fe3907b-a8ea-4d91-af2f-a263d4ab9c4c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-83978 namespace: kserve-ci-e2e-test resourceVersion: "22815" uid: 8e3b5190-3bb5-4ba3-b569-9d7ab526367e kind: Event lastTimestamp: "2026-04-23T18:23:56Z" message: InferenceService [success-200-isvc-83978] is Ready metadata: creationTimestamp: "2026-04-23T18:23:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:56Z" name: success-200-isvc-83978.18a90f8f7e738f6a namespace: kserve-ci-e2e-test resourceVersion: "22816" uid: 182fa74e-2180-4f45-8a96-f786e39a1e67 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:19.607778Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17502" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-89c6b-predictor-9f5f766-8s6w4 to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:19Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90efb19dbc957 namespace: kserve-ci-e2e-test resourceVersion: "17514" uid: 854dba5e-438b-4b3d-a289-947be989cc2c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:20Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90efb33ab363d namespace: kserve-ci-e2e-test resourceVersion: "17573" uid: 0f73026a-7f97-4240-a78a-d4d7ca7aba61 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90efb3519518a namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: 0dccee17-448c-4331-904b-6e3f6740640f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90efb399c912e namespace: kserve-ci-e2e-test resourceVersion: "17578" uid: 7a167fb0-23ff-43dc-b56f-fd5c8eeca958 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90efb3a11da6a namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: 8c6df3ed-a8f8-44d8-be3f-018faaf051f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90efb3a1ed3fc namespace: kserve-ci-e2e-test resourceVersion: "17581" uid: f879ecbc-2af3-4604-8453-f460ceeb2801 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90efb3d35fe1d namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 7693b3e9-6b60-4e7c-b0b6-893a20258bea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:13:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:20Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90efb3dacf7a6 namespace: kserve-ci-e2e-test resourceVersion: "17585" uid: cc8b6805-05f3-4b91-af4a-778ecdfda9f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T18:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:22:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:47Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90efba028684d namespace: kserve-ci-e2e-test resourceVersion: "22006" uid: 9609af2d-74b8-4c38-bd48-3ceec85e5ee9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:44Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90f7e9ef2e171 namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: f9270971-5277-40eb-a76c-709193a9f93a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:44Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90f7e9ef3a69f namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: 6b732bd8-550e-474a-8345-632f6752767c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 63aab03a-546c-4970-b61d-5e334d73e842 kind: Event lastTimestamp: "2026-04-23T18:22:47Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:47Z" name: success-200-isvc-89c6b-predictor-9f5f766-8s6w4.18a90f7f68919c5e namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 131d848c-13e7-4704-8eb0-739bc73af202 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-89c6b-predictor-9f5f766 namespace: kserve-ci-e2e-test resourceVersion: "17483" uid: 78d94dfa-28a6-4b8f-9ba3-39eb719294fb kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: 'Created pod: success-200-isvc-89c6b-predictor-9f5f766-8s6w4' metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:19Z" name: success-200-isvc-89c6b-predictor-9f5f766.18a90efb19604779 namespace: kserve-ci-e2e-test resourceVersion: "17505" uid: fc9afaba-8402-44fd-9931-70384df0d342 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17478" uid: b372a851-51b9-4bfd-992d-f830f3dbce58 kind: Event lastTimestamp: "2026-04-23T18:13:19Z" message: Scaled up replica set success-200-isvc-89c6b-predictor-9f5f766 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:19Z" name: success-200-isvc-89c6b-predictor.18a90efb178a80c7 namespace: kserve-ci-e2e-test resourceVersion: "17487" uid: 5cc8bfce-a079-4c8f-b329-3825c69df39a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17482" uid: 5332c4fe-a544-4329-9b7d-3566a8e02064 kind: Event lastTimestamp: "2026-04-23T18:13:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:49Z" name: success-200-isvc-89c6b-predictor.18a90efe962377d3 namespace: kserve-ci-e2e-test resourceVersion: "17905" uid: c676526d-bbbc-42bc-be89-f6a4f5eca01c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17482" uid: 5332c4fe-a544-4329-9b7d-3566a8e02064 kind: Event lastTimestamp: "2026-04-23T18:13:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:49Z" name: success-200-isvc-89c6b-predictor.18a90efe96d8bb13 namespace: kserve-ci-e2e-test resourceVersion: "17906" uid: bffbf4ba-e2ec-4e26-8d46-6e85236e7ee2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:14:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17749" uid: 5332c4fe-a544-4329-9b7d-3566a8e02064 kind: Event lastTimestamp: "2026-04-23T18:14:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:34Z" name: success-200-isvc-89c6b-predictor.18a90f05945386e0 namespace: kserve-ci-e2e-test resourceVersion: "18301" uid: fca420d6-3def-4f4e-a06b-f264b9609058 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:14:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "17749" uid: 5332c4fe-a544-4329-9b7d-3566a8e02064 kind: Event lastTimestamp: "2026-04-23T18:14:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:34Z" name: success-200-isvc-89c6b-predictor.18a90f0594a42687 namespace: kserve-ci-e2e-test resourceVersion: "18302" uid: 50427f5f-a9f7-4440-bb59-52152464b660 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:13:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89c6b namespace: kserve-ci-e2e-test resourceVersion: "17465" uid: 68586d1b-9398-441e-9d68-9e88d97be649 kind: Event lastTimestamp: "2026-04-23T18:14:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:18Z" name: success-200-isvc-89c6b.18a90efb175b36be namespace: kserve-ci-e2e-test resourceVersion: "18146" uid: 78ca96d6-d239-4eb4-b1e4-580aa9766fb4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89c6b namespace: kserve-ci-e2e-test resourceVersion: "18143" uid: 68586d1b-9398-441e-9d68-9e88d97be649 kind: Event lastTimestamp: "2026-04-23T18:14:18Z" message: InferenceService [success-200-isvc-89c6b] is Ready metadata: creationTimestamp: "2026-04-23T18:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:14:18Z" name: success-200-isvc-89c6b.18a90f08b2f14ee8 namespace: kserve-ci-e2e-test resourceVersion: "18144" uid: f3a7e46b-1c67-48b8-bcf1-0166f7840306 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:22:44.530313Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-89ff7-predictor-55645f8766-rbr8g to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:22:44Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7ea1dd95a8 namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: 79361693-8d51-4f74-9f0c-283776c366b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-89ff7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:44Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7eaa1b282b namespace: kserve-ci-e2e-test resourceVersion: "21926" uid: bb3c83fb-79a4-4a5a-8513-8c2007f3f3af reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21901" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:22:45Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7ee013e6be namespace: kserve-ci-e2e-test resourceVersion: "21949" uid: 3f1a40d2-44d0-4ae9-b927-50d69738851c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7ee16f230a namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: 895d7018-bf38-4fe5-8b7e-52fe0e3caa3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7ee625d7f7 namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: f11d9ce0-3235-439f-b27b-bf5c67a25447 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7ee69c7e91 namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: eeca0e1e-684d-4e02-b4b3-8b999feab684 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7ee6aaed8c namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: 861dcd06-cc22-4cba-8ffa-604f59a602fa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7ee9a60814 namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: 587b8977-8bf0-4eef-a626-ae8675c1d782 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:22:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:45Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7eea30bab7 namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 51f18fa0-c8aa-42c1-bbe2-7ea4470987c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:22:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:23:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:23:32Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f7f27ba0635 namespace: kserve-ci-e2e-test resourceVersion: "22569" uid: 81349e74-5b74-420b-a1e4-d0526280ab8a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:04Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f914ec4a9ca namespace: kserve-ci-e2e-test resourceVersion: "22926" uid: b2e15b85-fff8-4b25-9dfe-cc2f2b0e052b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:24:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:04Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f914ec5c8f4 namespace: kserve-ci-e2e-test resourceVersion: "22929" uid: 372ef3d3-b1b2-4a09-96ac-437ca184a8a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:24:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 30ec9c27-aca5-42c9-ac1d-c7c522cdf327 kind: Event lastTimestamp: "2026-04-23T18:24:07Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:24:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:24:07Z" name: success-200-isvc-89ff7-predictor-55645f8766-rbr8g.18a90f9203e0383e namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: fd0320b1-8932-4822-aadf-2b5d8e40bdf7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-89ff7-predictor-55645f8766 namespace: kserve-ci-e2e-test resourceVersion: "21867" uid: e36fd522-e805-4cbe-aac0-50f180556653 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: 'Created pod: success-200-isvc-89ff7-predictor-55645f8766-rbr8g' metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:44Z" name: success-200-isvc-89ff7-predictor-55645f8766.18a90f7ea1558f20 namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: bdd7ee0d-7c73-4b35-9b2f-57fd0565a99a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21864" uid: ddccbff6-42db-4c34-aa91-2f847af9183c kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: Scaled up replica set success-200-isvc-89ff7-predictor-55645f8766 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:44Z" name: success-200-isvc-89ff7-predictor.18a90f7e9f814aa4 namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: 9f6fbfcd-a059-408d-8141-7cd65835e155 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21866" uid: 98221271-e4dc-4e93-bc4e-e68797254861 kind: Event lastTimestamp: "2026-04-23T18:22:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:59Z" name: success-200-isvc-89ff7-predictor.18a90f821df33edd namespace: kserve-ci-e2e-test resourceVersion: "22135" uid: 7d41d0d2-dd4c-4dc7-b7e4-a7734f5b925f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21866" uid: 98221271-e4dc-4e93-bc4e-e68797254861 kind: Event lastTimestamp: "2026-04-23T18:22:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:22:59Z" name: success-200-isvc-89ff7-predictor.18a90f821e6480e7 namespace: kserve-ci-e2e-test resourceVersion: "22137" uid: 3bc1ae98-0141-4d77-91ea-d93e437e4317 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:23:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22136" uid: 98221271-e4dc-4e93-bc4e-e68797254861 kind: Event lastTimestamp: "2026-04-23T18:23:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:59Z" name: success-200-isvc-89ff7-predictor.18a90f859d17c999 namespace: kserve-ci-e2e-test resourceVersion: "22836" uid: 354107db-3634-42ef-bd5a-222db3d4de16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:23:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89ff7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22136" uid: 98221271-e4dc-4e93-bc4e-e68797254861 kind: Event lastTimestamp: "2026-04-23T18:23:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:23:59Z" name: success-200-isvc-89ff7-predictor.18a90f859d7f8bed namespace: kserve-ci-e2e-test resourceVersion: "22837" uid: fc6e13cc-e51c-4a99-92af-ffd979805dfe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 7730a7f4-9e9c-434e-8f18-273f379241d4 kind: Event lastTimestamp: "2026-04-23T18:23:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:42Z" name: success-200-isvc-89ff7.18a90f7e9f48200f namespace: kserve-ci-e2e-test resourceVersion: "22664" uid: d471bd79-e3eb-4b11-b733-648c31d7560c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 7730a7f4-9e9c-434e-8f18-273f379241d4 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-89ff7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-89ff7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:44Z" name: success-200-isvc-89ff7.18a90f7e9f95eac3 namespace: kserve-ci-e2e-test resourceVersion: "21872" uid: f1e18497-a117-4b8f-99a1-3972401d0e83 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 7730a7f4-9e9c-434e-8f18-273f379241d4 kind: Event lastTimestamp: "2026-04-23T18:22:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-89ff7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:22:44Z" name: success-200-isvc-89ff7.18a90f7e9f962066 namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: 0758e849-e507-4413-9c9c-5967d05cf3aa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:23:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89ff7 namespace: kserve-ci-e2e-test resourceVersion: "22662" uid: 7730a7f4-9e9c-434e-8f18-273f379241d4 kind: Event lastTimestamp: "2026-04-23T18:23:42Z" message: InferenceService [success-200-isvc-89ff7] is Ready metadata: creationTimestamp: "2026-04-23T18:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:23:42Z" name: success-200-isvc-89ff7.18a90f8c367e5006 namespace: kserve-ci-e2e-test resourceVersion: "22663" uid: a257054c-8867-4754-a764-d484fbc545a1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:10:59.564946Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-be430-predictor-7c764dd68b-m4lt6 to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:10:59Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90eda7ea7c713 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 33cbcbba-23cd-4f63-961c-d8eb7eae943a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-be430-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:10:59Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90eda8c5fe05d namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 0184e4d9-4f08-4225-87c6-2f09be31b95b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:00Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:11:00Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edabc165a92 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: 7a5e6f29-57f8-44ca-b7ea-399a3471bee7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:00Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" metadata: creationTimestamp: "2026-04-23T18:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:00Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edabddbeec1 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 50fa97a1-5d46-4821-ae4e-8c5d1bf064dd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" in 17.948s (17.948s including waiting). Image size: 1337676403 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edeebb2b7b2 namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: 508c4f68-40e1-4db6-9ea0-668acb85c428 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edeefd8b61e namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: caa23725-4057-4ca2-a520-b662d92e4bac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edef05d2883 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 78b7f282-e17f-4bde-a903-b22510075a06 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T18:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:18Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edef06d331e namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: ba19aaf5-eb9a-4fe9-9b8d-11bcc4b6f250 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:21Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.574s (2.574s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:21Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edf89e4c1e7 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: dec3918d-0a84-4776-80c6-daac46c896a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:21Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edf8e5d1eaf namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 98a9651f-a135-433c-ab3a-88cdb54cc703 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:21Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edf8ed4ebdb namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 0b558e74-cb37-49b3-84fe-b79e69219a43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:11:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:11:58Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90edfd2246910 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: ca697049-4d97-4f9a-93ee-f3a2bf5a9dc3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:33Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90ef0669a81b5 namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 15b7d4b9-6535-46a5-85c8-0750170164f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: d9103716-7953-41cb-aa5a-fcd84af6c727 kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:33Z" name: success-200-isvc-be430-predictor-7c764dd68b-m4lt6.18a90ef0669aef22 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: 29442ebe-18e0-432e-90a6-1585d014d069 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-be430-predictor-7c764dd68b namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: cca5e5e8-5f90-41e8-9346-7f7dd05772c0 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: 'Created pod: success-200-isvc-be430-predictor-7c764dd68b-m4lt6' metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:10:59Z" name: success-200-isvc-be430-predictor-7c764dd68b.18a90eda7df9e338 namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: 94d14174-bab3-4ad2-b916-ca2970114659 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 62bf1a41-d82b-4a37-9c78-ee55746efec3 kind: Event lastTimestamp: "2026-04-23T18:10:59Z" message: Scaled up replica set success-200-isvc-be430-predictor-7c764dd68b from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:10:59Z" name: success-200-isvc-be430-predictor.18a90eda7c33fa9a namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 4bbb35a8-87e5-4c11-8fbf-fee9c880707e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 290cf7d9-4f61-4ef6-a36f-413ccfb94efe kind: Event lastTimestamp: "2026-04-23T18:11:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:59Z" name: success-200-isvc-be430-predictor.18a90eddfadae439 namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: a480425e-5083-4011-8487-3fe26d7ab684 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 290cf7d9-4f61-4ef6-a36f-413ccfb94efe kind: Event lastTimestamp: "2026-04-23T18:11:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:11:59Z" name: success-200-isvc-be430-predictor.18a90eddfb4af8d8 namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: 7e0c499c-b5aa-407c-bb20-04d5f74bf6a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 290cf7d9-4f61-4ef6-a36f-413ccfb94efe kind: Event lastTimestamp: "2026-04-23T18:12:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:29Z" name: success-200-isvc-be430-predictor.18a90ee4f8ca4451 namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: ed1024af-efca-4ffb-8609-ca959f957cfd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T18:11:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be430-predictor namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 290cf7d9-4f61-4ef6-a36f-413ccfb94efe kind: Event lastTimestamp: "2026-04-23T18:12:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:29Z" name: success-200-isvc-be430-predictor.18a90ee4f934f96e namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 84b82cdf-677a-4c28-ac0e-3013d24ff257 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:10:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be430 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 8ef4e9e2-652a-4331-bf0b-f380ee86f9cd kind: Event lastTimestamp: "2026-04-23T18:12:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:10:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:08Z" name: success-200-isvc-be430.18a90eda7c08b4bf namespace: kserve-ci-e2e-test resourceVersion: "16605" uid: 7a4fad36-9c93-47de-9604-dbcf24022e8e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be430 namespace: kserve-ci-e2e-test resourceVersion: "16603" uid: 8ef4e9e2-652a-4331-bf0b-f380ee86f9cd kind: Event lastTimestamp: "2026-04-23T18:12:08Z" message: InferenceService [success-200-isvc-be430] is Ready metadata: creationTimestamp: "2026-04-23T18:12:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:08Z" name: success-200-isvc-be430.18a90eea8cfc0abe namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: 9da46d55-d026-49cc-bfb1-cebdbcaf0891 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:33:29.663050Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27690" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-febb8-predictor-7b967b5c88-c75hz to ip-10-0-143-131.ec2.internal metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:33:29Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a91014d6c5f77a namespace: kserve-ci-e2e-test resourceVersion: "27699" uid: aecc0046-157d-499d-856a-073ac699019e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-febb8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:29Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a91014e48f46b9 namespace: kserve-ci-e2e-test resourceVersion: "27739" uid: 2806a7fc-4bc6-4257-b416-da2e996d621e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27709" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:33:30Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a9101514480e3f namespace: kserve-ci-e2e-test resourceVersion: "27756" uid: e73f2b1b-487a-4a49-a941-b965ea312afc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1419" already present on machine metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a9101515a002a9 namespace: kserve-ci-e2e-test resourceVersion: "27758" uid: fb203380-46ad-46b5-b2c4-0104ab42245f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a910151a59c949 namespace: kserve-ci-e2e-test resourceVersion: "27760" uid: a81cbab6-fd42-4d0a-a7b2-42a98158a3e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a910151af4a217 namespace: kserve-ci-e2e-test resourceVersion: "27761" uid: 0c7ca9f1-b557-43e2-a664-cefe64b6db01 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a910151b04bdbd namespace: kserve-ci-e2e-test resourceVersion: "27762" uid: 83235773-f038-40bc-a4a2-c52cc61cce9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a910151f47402e namespace: kserve-ci-e2e-test resourceVersion: "27766" uid: 0ee365e9-4ed8-4f03-a4ee-5cc3557c87aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:33:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:33:30Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a910151fc38aac namespace: kserve-ci-e2e-test resourceVersion: "27767" uid: 2e0477cf-a436-4ed5-908a-b46c871458fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T18:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:34:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T18:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:18Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a9101565803db2 namespace: kserve-ci-e2e-test resourceVersion: "28328" uid: 1b6a2d86-921b-4783-959e-4a2dc119de23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:50:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T18:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:50:59Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a911093279d482 namespace: kserve-ci-e2e-test resourceVersion: "35385" uid: 824aec56-5feb-402d-beba-204792948875 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 181be6bb-ce95-4aa3-b958-c28a19b93cee kind: Event lastTimestamp: "2026-04-23T18:50:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T18:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:50:59Z" name: success-200-isvc-febb8-predictor-7b967b5c88-c75hz.18a91109327a94a8 namespace: kserve-ci-e2e-test resourceVersion: "35386" uid: a5e8190c-525a-4a30-a742-37f696cc4684 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-131.ec2.internal source: component: kubelet host: ip-10-0-143-131.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-febb8-predictor-7b967b5c88 namespace: kserve-ci-e2e-test resourceVersion: "27678" uid: 54e2aaf9-081c-4a67-8c84-8ad24258eec0 kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: 'Created pod: success-200-isvc-febb8-predictor-7b967b5c88-c75hz' metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:29Z" name: success-200-isvc-febb8-predictor-7b967b5c88.18a91014d6244a84 namespace: kserve-ci-e2e-test resourceVersion: "27692" uid: 3e70fc00-01a5-4118-96b2-06de6e2315fb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "27675" uid: a7ee37bd-4f64-4db9-9c40-6322f5e09cd1 kind: Event lastTimestamp: "2026-04-23T18:33:29Z" message: Scaled up replica set success-200-isvc-febb8-predictor-7b967b5c88 from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:29Z" name: success-200-isvc-febb8-predictor.18a91014d3deb09c namespace: kserve-ci-e2e-test resourceVersion: "27683" uid: 463042ea-b774-483a-a000-8d8c4fed0046 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "27677" uid: b30e8abf-57d0-4d8f-a295-f8a7abbe3c7f kind: Event lastTimestamp: "2026-04-23T18:33:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:33:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:44Z" name: success-200-isvc-febb8-predictor.18a910185255702b namespace: kserve-ci-e2e-test resourceVersion: "28051" uid: c35ceedb-8ad7-4e53-8d30-232f3c238ac3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:33:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "27677" uid: b30e8abf-57d0-4d8f-a295-f8a7abbe3c7f kind: Event lastTimestamp: "2026-04-23T18:33:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:33:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:33:44Z" name: success-200-isvc-febb8-predictor.18a9101852befb39 namespace: kserve-ci-e2e-test resourceVersion: "28053" uid: c94f0b2f-530e-488e-97b3-3640ff10c5a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:33:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "28052" uid: b30e8abf-57d0-4d8f-a295-f8a7abbe3c7f kind: Event lastTimestamp: "2026-04-23T18:34:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:33:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:44Z" name: success-200-isvc-febb8-predictor.18a9101bd17b68e0 namespace: kserve-ci-e2e-test resourceVersion: "28646" uid: c62b4fae-7cb5-4752-922f-1249316043f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T18:33:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-febb8-predictor namespace: kserve-ci-e2e-test resourceVersion: "28052" uid: b30e8abf-57d0-4d8f-a295-f8a7abbe3c7f kind: Event lastTimestamp: "2026-04-23T18:34:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:33:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:44Z" name: success-200-isvc-febb8-predictor.18a9101bd1ea682f namespace: kserve-ci-e2e-test resourceVersion: "28647" uid: 1b51b3b6-5cf7-4c09-9f1e-b10b54b8c40c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:33:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-febb8 namespace: kserve-ci-e2e-test resourceVersion: "27656" uid: 2c87273e-68ae-4a55-b738-7efc4b5bd3a8 kind: Event lastTimestamp: "2026-04-23T18:34:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T18:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:28Z" name: success-200-isvc-febb8.18a91014d39f7ecb namespace: kserve-ci-e2e-test resourceVersion: "28431" uid: 5358976d-3101-408c-b195-9ea8369ef547 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-febb8 namespace: kserve-ci-e2e-test resourceVersion: "28429" uid: 2c87273e-68ae-4a55-b738-7efc4b5bd3a8 kind: Event lastTimestamp: "2026-04-23T18:34:28Z" message: InferenceService [success-200-isvc-febb8] is Ready metadata: creationTimestamp: "2026-04-23T18:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:28Z" name: success-200-isvc-febb8.18a91022751a76e5 namespace: kserve-ci-e2e-test resourceVersion: "28430" uid: 58392a13-fe05-4bd2-8880-17c9c74f1e69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:13:43.801120Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-03167-6b54457b8f-b6sgt namespace: kserve-ci-e2e-test resourceVersion: "17832" uid: 06345e49-9db5-44f6-bf83-cbf0714e3aca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-03167-6b54457b8f-b6sgt to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:13:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:13:43Z" name: switch-graph-03167-6b54457b8f-b6sgt.18a90f00bbe4d865 namespace: kserve-ci-e2e-test resourceVersion: "17837" uid: b2dd6e1c-25ce-44ef-80d5-0af621c14911 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:44Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-03167-6b54457b8f-b6sgt namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: 06345e49-9db5-44f6-bf83-cbf0714e3aca kind: Event lastTimestamp: "2026-04-23T18:13:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-03167-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:44Z" name: switch-graph-03167-6b54457b8f-b6sgt.18a90f00c80a6ddd namespace: kserve-ci-e2e-test resourceVersion: "17849" uid: 6ab3bc7b-2cad-4f9a-bd75-f7450b319081 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:44Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-03167-6b54457b8f-b6sgt namespace: kserve-ci-e2e-test resourceVersion: "17842" uid: 06345e49-9db5-44f6-bf83-cbf0714e3aca kind: Event lastTimestamp: "2026-04-23T18:13:44Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:13:44Z" name: switch-graph-03167-6b54457b8f-b6sgt.18a90f00f853932d namespace: kserve-ci-e2e-test resourceVersion: "17859" uid: 5740f761-f4fd-4c6d-a370-fb1f23798c41 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-03167} kind: Pod name: switch-graph-03167-6b54457b8f-b6sgt namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: 06345e49-9db5-44f6-bf83-cbf0714e3aca kind: Event lastTimestamp: "2026-04-23T18:13:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:44Z" name: switch-graph-03167-6b54457b8f-b6sgt.18a90f00f9a7d7be namespace: kserve-ci-e2e-test resourceVersion: "17861" uid: f6d3016d-1fa0-41b6-8307-b95e2d01573c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-03167} kind: Pod name: switch-graph-03167-6b54457b8f-b6sgt namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: 06345e49-9db5-44f6-bf83-cbf0714e3aca kind: Event lastTimestamp: "2026-04-23T18:13:44Z" message: 'Created container: switch-graph-03167' metadata: creationTimestamp: "2026-04-23T18:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:44Z" name: switch-graph-03167-6b54457b8f-b6sgt.18a90f00fcbad15e namespace: kserve-ci-e2e-test resourceVersion: "17862" uid: 33705cb1-bd9d-4121-8b02-24e6541cf625 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-03167} kind: Pod name: switch-graph-03167-6b54457b8f-b6sgt namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: 06345e49-9db5-44f6-bf83-cbf0714e3aca kind: Event lastTimestamp: "2026-04-23T18:13:44Z" message: Started container switch-graph-03167 metadata: creationTimestamp: "2026-04-23T18:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:13:44Z" name: switch-graph-03167-6b54457b8f-b6sgt.18a90f00fd283bb4 namespace: kserve-ci-e2e-test resourceVersion: "17863" uid: a521465d-a772-46e1-aa0e-9a4096eef0ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-03167} kind: Pod name: switch-graph-03167-6b54457b8f-b6sgt namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: 06345e49-9db5-44f6-bf83-cbf0714e3aca kind: Event lastTimestamp: "2026-04-23T18:21:58Z" message: Stopping container switch-graph-03167 metadata: creationTimestamp: "2026-04-23T18:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:21:58Z" name: switch-graph-03167-6b54457b8f-b6sgt.18a90f73ea4fc413 namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: f36e3fb3-2304-4111-8922-57ddcd414a9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-03167} kind: Pod name: switch-graph-03167-6b54457b8f-b6sgt namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: 06345e49-9db5-44f6-bf83-cbf0714e3aca kind: Event lastTimestamp: "2026-04-23T18:22:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:22:26Z" name: switch-graph-03167-6b54457b8f-b6sgt.18a90f74805843e4 namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: 94c0b3fe-9a33-41b7-b15f-42a740377b75 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-03167-6b54457b8f namespace: kserve-ci-e2e-test resourceVersion: "17829" uid: 6dd1b9b3-cd0f-4779-a468-a2ce593ec1a9 kind: Event lastTimestamp: "2026-04-23T18:13:43Z" message: 'Created pod: switch-graph-03167-6b54457b8f-b6sgt' metadata: creationTimestamp: "2026-04-23T18:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:43Z" name: switch-graph-03167-6b54457b8f.18a90f00bb63a3f0 namespace: kserve-ci-e2e-test resourceVersion: "17834" uid: f0e23bf6-20ba-4113-b6f7-644d0083f114 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-03167 namespace: kserve-ci-e2e-test resourceVersion: "17827" uid: 05ef5363-ff79-4ce7-be6d-203bbfc0f50f kind: Event lastTimestamp: "2026-04-23T18:13:43Z" message: Scaled up replica set switch-graph-03167-6b54457b8f from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:43Z" name: switch-graph-03167.18a90f00ba26a750 namespace: kserve-ci-e2e-test resourceVersion: "17830" uid: b913c2f4-287c-4540-bd05-645400e3c2e4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-03167 namespace: kserve-ci-e2e-test resourceVersion: "17844" uid: 6566db59-4ef9-45f1-9043-6e301e70e509 kind: Event lastTimestamp: "2026-04-23T18:13:43Z" message: InferenceGraph [switch-graph-03167] is Ready metadata: creationTimestamp: "2026-04-23T18:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:13:43Z" name: switch-graph-03167.18a90f00bfc75d60 namespace: kserve-ci-e2e-test resourceVersion: "17846" uid: 8c734da5-ceaa-4f39-b977-6f31df2e6b80 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-03167 namespace: kserve-ci-e2e-test resourceVersion: "17828" uid: 02eb9eb9-f802-46c5-a952-82bc9db0e917 kind: Event lastTimestamp: "2026-04-23T18:13:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:58Z" name: switch-graph-03167.18a90f0438d314d8 namespace: kserve-ci-e2e-test resourceVersion: "17982" uid: 3cddc913-73cc-487f-83a9-32c0f70d99a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:13:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-03167 namespace: kserve-ci-e2e-test resourceVersion: "17828" uid: 02eb9eb9-f802-46c5-a952-82bc9db0e917 kind: Event lastTimestamp: "2026-04-23T18:13:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:13:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:13:58Z" name: switch-graph-03167.18a90f04394a59f2 namespace: kserve-ci-e2e-test resourceVersion: "17985" uid: 8fbaeb50-49dd-4c59-aa70-4cb030e7b59a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-03167 namespace: kserve-ci-e2e-test resourceVersion: "17983" uid: 02eb9eb9-f802-46c5-a952-82bc9db0e917 kind: Event lastTimestamp: "2026-04-23T18:14:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:13Z" name: switch-graph-03167.18a90f07b800e7e9 namespace: kserve-ci-e2e-test resourceVersion: "18100" uid: 6323ea7e-d40b-471a-9002-1670a5c07315 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:14:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-03167 namespace: kserve-ci-e2e-test resourceVersion: "17983" uid: 02eb9eb9-f802-46c5-a952-82bc9db0e917 kind: Event lastTimestamp: "2026-04-23T18:14:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T18:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:14:13Z" name: switch-graph-03167.18a90f07b850dab5 namespace: kserve-ci-e2e-test resourceVersion: "18101" uid: e2b53d8b-306f-4bf9-9b84-1324981ddbce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:12:19.389043Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-be430-657bc8887b-gmv4m namespace: kserve-ci-e2e-test resourceVersion: "16714" uid: 94cc07fa-51f6-458a-868d-d9f708940d34 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-be430-657bc8887b-gmv4m to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:12:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:12:19Z" name: switch-graph-be430-657bc8887b-gmv4m.18a90eed148ad64a namespace: kserve-ci-e2e-test resourceVersion: "16719" uid: 9596759c-3cf6-4b7a-8e7b-e52a513b4876 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:19Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-be430-657bc8887b-gmv4m namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 94cc07fa-51f6-458a-868d-d9f708940d34 kind: Event lastTimestamp: "2026-04-23T18:12:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-be430-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:19Z" name: switch-graph-be430-657bc8887b-gmv4m.18a90eed20b9b064 namespace: kserve-ci-e2e-test resourceVersion: "16733" uid: 8ae184cd-bdae-4b35-a47b-bb31a52ddf16 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-be430-657bc8887b-gmv4m namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 94cc07fa-51f6-458a-868d-d9f708940d34 kind: Event lastTimestamp: "2026-04-23T18:12:20Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:12:20Z" name: switch-graph-be430-657bc8887b-gmv4m.18a90eed52675df7 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: 79a86d95-df75-43e3-8092-c90890d7366e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be430} kind: Pod name: switch-graph-be430-657bc8887b-gmv4m namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 94cc07fa-51f6-458a-868d-d9f708940d34 kind: Event lastTimestamp: "2026-04-23T18:12:20Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" metadata: creationTimestamp: "2026-04-23T18:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:20Z" name: switch-graph-be430-657bc8887b-gmv4m.18a90eed53a3eeb0 namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: bd8b17f4-91f1-4a3e-8600-9ab9f6c43530 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be430} kind: Pod name: switch-graph-be430-657bc8887b-gmv4m namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 94cc07fa-51f6-458a-868d-d9f708940d34 kind: Event lastTimestamp: "2026-04-23T18:12:22Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" in 2.402s (2.402s including waiting). Image size: 216346205 bytes.' metadata: creationTimestamp: "2026-04-23T18:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:22Z" name: switch-graph-be430-657bc8887b-gmv4m.18a90eede2d8a587 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: 64a278b5-26b6-4329-992f-74709f01b537 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be430} kind: Pod name: switch-graph-be430-657bc8887b-gmv4m namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 94cc07fa-51f6-458a-868d-d9f708940d34 kind: Event lastTimestamp: "2026-04-23T18:12:22Z" message: 'Created container: switch-graph-be430' metadata: creationTimestamp: "2026-04-23T18:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:22Z" name: switch-graph-be430-657bc8887b-gmv4m.18a90eede640dd9c namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 55f89123-22cc-40fb-aaf8-ecf86069859d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be430} kind: Pod name: switch-graph-be430-657bc8887b-gmv4m namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 94cc07fa-51f6-458a-868d-d9f708940d34 kind: Event lastTimestamp: "2026-04-23T18:12:22Z" message: Started container switch-graph-be430 metadata: creationTimestamp: "2026-04-23T18:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:22Z" name: switch-graph-be430-657bc8887b-gmv4m.18a90eede6b47c4c namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 16206a65-8b67-43b3-9976-255335a7b306 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be430} kind: Pod name: switch-graph-be430-657bc8887b-gmv4m namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 94cc07fa-51f6-458a-868d-d9f708940d34 kind: Event lastTimestamp: "2026-04-23T18:12:33Z" message: Stopping container switch-graph-be430 metadata: creationTimestamp: "2026-04-23T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:33Z" name: switch-graph-be430-657bc8887b-gmv4m.18a90ef05f62545a namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: 19e998ab-4e10-49fc-8266-9d8cff2d2e47 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T18:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be430} kind: Pod name: switch-graph-be430-657bc8887b-gmv4m namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 94cc07fa-51f6-458a-868d-d9f708940d34 kind: Event lastTimestamp: "2026-04-23T18:12:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:12:59Z" name: switch-graph-be430-657bc8887b-gmv4m.18a90ef0a80d2de0 namespace: kserve-ci-e2e-test resourceVersion: "17240" uid: 7a228e88-ffd3-485c-88be-9d8bc6137ed0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-be430-657bc8887b namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 340fbd8d-29d5-4b6e-9844-7e7180f6e2c9 kind: Event lastTimestamp: "2026-04-23T18:12:19Z" message: 'Created pod: switch-graph-be430-657bc8887b-gmv4m' metadata: creationTimestamp: "2026-04-23T18:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:19Z" name: switch-graph-be430-657bc8887b.18a90eed1430b03d namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: d5799439-4703-486d-a40e-100dcc7352a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-be430 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: dd129944-7bb1-47c1-a3dd-bb7622431110 kind: Event lastTimestamp: "2026-04-23T18:12:19Z" message: Scaled up replica set switch-graph-be430-657bc8887b from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:12:19Z" name: switch-graph-be430.18a90eed13249944 namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: 1ef5d616-732a-44fe-90d8-ffd1d07c3768 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-be430 namespace: kserve-ci-e2e-test resourceVersion: "16725" uid: 9a65f0d8-a673-43d6-8c6d-f99b1541a0f1 kind: Event lastTimestamp: "2026-04-23T18:12:19Z" message: InferenceGraph [switch-graph-be430] is Ready metadata: creationTimestamp: "2026-04-23T18:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:19Z" name: switch-graph-be430.18a90eed1895069c namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 746227f9-816b-4cbe-aff2-37a06154cf00 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-be430 namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: 9a65f0d8-a673-43d6-8c6d-f99b1541a0f1 kind: Event lastTimestamp: "2026-04-23T18:12:19Z" message: 'Failed to update status for InferenceGraph "switch-graph-be430": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-be430": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:19Z" name: switch-graph-be430.18a90eed1aca20de namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 39de5e23-f665-4fbe-b7b5-b36534dd9cc0 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:12:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-be430 namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: 9a65f0d8-a673-43d6-8c6d-f99b1541a0f1 kind: Event lastTimestamp: "2026-04-23T18:12:19Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-be430": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T18:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:12:19Z" name: switch-graph-be430.18a90eed1aca64ae namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: 186bdd8b-11af-4b90-9774-78d126cdb62f reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T18:34:39.707417Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-febb8-7bb468f96c-5bnl7 namespace: kserve-ci-e2e-test resourceVersion: "28584" uid: e49426a4-15c8-4789-8200-264dc7c9c076 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-febb8-7bb468f96c-5bnl7 to ip-10-0-130-162.ec2.internal metadata: creationTimestamp: "2026-04-23T18:34:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T18:34:39Z" name: switch-graph-febb8-7bb468f96c-5bnl7.18a9102525be1fe9 namespace: kserve-ci-e2e-test resourceVersion: "28589" uid: d2748940-8a70-4c8a-b659-76f5c8e7c453 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-779b58ffc9-swn7z source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:39Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-febb8-7bb468f96c-5bnl7 namespace: kserve-ci-e2e-test resourceVersion: "28586" uid: e49426a4-15c8-4789-8200-264dc7c9c076 kind: Event lastTimestamp: "2026-04-23T18:34:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-febb8-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T18:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:39Z" name: switch-graph-febb8-7bb468f96c-5bnl7.18a910252f6af9c2 namespace: kserve-ci-e2e-test resourceVersion: "28603" uid: ab57a76a-5532-4b12-920b-2c575d8026b5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:40Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-febb8-7bb468f96c-5bnl7 namespace: kserve-ci-e2e-test resourceVersion: "28592" uid: e49426a4-15c8-4789-8200-264dc7c9c076 kind: Event lastTimestamp: "2026-04-23T18:34:40Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T18:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T18:34:40Z" name: switch-graph-febb8-7bb468f96c-5bnl7.18a910256259739a namespace: kserve-ci-e2e-test resourceVersion: "28613" uid: b4343533-550e-4d93-a024-4ade9c5ba453 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-febb8} kind: Pod name: switch-graph-febb8-7bb468f96c-5bnl7 namespace: kserve-ci-e2e-test resourceVersion: "28586" uid: e49426a4-15c8-4789-8200-264dc7c9c076 kind: Event lastTimestamp: "2026-04-23T18:34:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f78a4a7e96d454cf05b867e1bd6acbb3ca248da49c31bd1b18647d18fbde53b4" already present on machine metadata: creationTimestamp: "2026-04-23T18:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:40Z" name: switch-graph-febb8-7bb468f96c-5bnl7.18a91025637e3ff0 namespace: kserve-ci-e2e-test resourceVersion: "28615" uid: 9b4c3ba0-a9c1-44df-ae80-44e2f9584aba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-febb8} kind: Pod name: switch-graph-febb8-7bb468f96c-5bnl7 namespace: kserve-ci-e2e-test resourceVersion: "28586" uid: e49426a4-15c8-4789-8200-264dc7c9c076 kind: Event lastTimestamp: "2026-04-23T18:34:40Z" message: 'Created container: switch-graph-febb8' metadata: creationTimestamp: "2026-04-23T18:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:40Z" name: switch-graph-febb8-7bb468f96c-5bnl7.18a91025663a0422 namespace: kserve-ci-e2e-test resourceVersion: "28617" uid: fb9df9dd-8c19-4117-9ec8-9f41e717a659 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-febb8} kind: Pod name: switch-graph-febb8-7bb468f96c-5bnl7 namespace: kserve-ci-e2e-test resourceVersion: "28586" uid: e49426a4-15c8-4789-8200-264dc7c9c076 kind: Event lastTimestamp: "2026-04-23T18:34:40Z" message: Started container switch-graph-febb8 metadata: creationTimestamp: "2026-04-23T18:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:34:40Z" name: switch-graph-febb8-7bb468f96c-5bnl7.18a9102566aa8ca8 namespace: kserve-ci-e2e-test resourceVersion: "28618" uid: 124fc936-70eb-4537-9fac-0e726e1de079 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-febb8} kind: Pod name: switch-graph-febb8-7bb468f96c-5bnl7 namespace: kserve-ci-e2e-test resourceVersion: "28586" uid: e49426a4-15c8-4789-8200-264dc7c9c076 kind: Event lastTimestamp: "2026-04-23T18:50:59Z" message: Stopping container switch-graph-febb8 metadata: creationTimestamp: "2026-04-23T18:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:50:59Z" name: switch-graph-febb8-7bb468f96c-5bnl7.18a911092b53abeb namespace: kserve-ci-e2e-test resourceVersion: "35370" uid: c5d064cf-0241-466c-b7cc-90778ce0c238 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T18:51:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-febb8} kind: Pod name: switch-graph-febb8-7bb468f96c-5bnl7 namespace: kserve-ci-e2e-test resourceVersion: "28586" uid: e49426a4-15c8-4789-8200-264dc7c9c076 kind: Event lastTimestamp: "2026-04-23T18:51:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T18:51:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T18:51:22Z" name: switch-graph-febb8-7bb468f96c-5bnl7.18a9110a0733470c namespace: kserve-ci-e2e-test resourceVersion: "35613" uid: 62f0fb85-b92f-4acb-be60-df5abe84b833 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-162.ec2.internal source: component: kubelet host: ip-10-0-130-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-febb8-7bb468f96c namespace: kserve-ci-e2e-test resourceVersion: "28581" uid: c16037e9-1369-43da-a6e2-e9caf1707668 kind: Event lastTimestamp: "2026-04-23T18:34:39Z" message: 'Created pod: switch-graph-febb8-7bb468f96c-5bnl7' metadata: creationTimestamp: "2026-04-23T18:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:39Z" name: switch-graph-febb8-7bb468f96c.18a910252556ba3e namespace: kserve-ci-e2e-test resourceVersion: "28585" uid: 8baf8816-1387-47a2-bbd2-57fec7bfe0b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-febb8 namespace: kserve-ci-e2e-test resourceVersion: "28579" uid: 6ef36e6c-bd22-40f0-a5a1-d5f8970d9942 kind: Event lastTimestamp: "2026-04-23T18:34:39Z" message: Scaled up replica set switch-graph-febb8-7bb468f96c from 0 to 1 metadata: creationTimestamp: "2026-04-23T18:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:34:39Z" name: switch-graph-febb8.18a910252498c3d2 namespace: kserve-ci-e2e-test resourceVersion: "28582" uid: bd382499-2153-4a8e-8f9c-8445dcc4d891 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T18:34:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-febb8 namespace: kserve-ci-e2e-test resourceVersion: "28595" uid: b40f269a-6d21-4e7d-8b87-30c90d27b42b kind: Event lastTimestamp: "2026-04-23T18:34:39Z" message: InferenceGraph [switch-graph-febb8] is Ready metadata: creationTimestamp: "2026-04-23T18:34:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T18:34:39Z" name: switch-graph-febb8.18a9102527d71810 namespace: kserve-ci-e2e-test resourceVersion: "28598" uid: a6cfd349-8971-4776-afd2-8f9a605f2181 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:34:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-febb8 namespace: kserve-ci-e2e-test resourceVersion: "28580" uid: 85d004a4-3b5f-4722-b0cb-2eaea499b14d kind: Event lastTimestamp: "2026-04-23T18:35:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:35:09Z" name: switch-graph-febb8.18a91028a3206748 namespace: kserve-ci-e2e-test resourceVersion: "28840" uid: fc9082e7-01df-40ef-ac30-7f015a0145d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T18:34:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-febb8 namespace: kserve-ci-e2e-test resourceVersion: "28580" uid: 85d004a4-3b5f-4722-b0cb-2eaea499b14d kind: Event lastTimestamp: "2026-04-23T18:35:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T18:34:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T18:35:09Z" name: switch-graph-febb8.18a91028a36a5b09 namespace: kserve-ci-e2e-test resourceVersion: "28841" uid: 842323c0-9df5-4c8d-8c82-e28d97c03395 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "35613"