--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:38:15.154044Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-091ad-8999dbf5b-s5rpc namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 003dfaa6-16d1-4f32-85d6-5d114a54c1fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-091ad-8999dbf5b-s5rpc to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:38:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:38:15Z" name: ensemble-graph-091ad-8999dbf5b-s5rpc.18a6ea475633caa3 namespace: kserve-ci-e2e-test resourceVersion: "19585" uid: 9d9045ff-b130-4ec4-9880-c88a4ad145dd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:15Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-091ad-8999dbf5b-s5rpc namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 003dfaa6-16d1-4f32-85d6-5d114a54c1fa kind: Event lastTimestamp: "2026-04-16T18:38:15Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:38:15Z" name: ensemble-graph-091ad-8999dbf5b-s5rpc.18a6ea476eb1b4a1 namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: e0240f73-0ed4-4007-bd22-66a71b85960a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-091ad} kind: Pod name: ensemble-graph-091ad-8999dbf5b-s5rpc namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: 003dfaa6-16d1-4f32-85d6-5d114a54c1fa kind: Event lastTimestamp: "2026-04-16T18:38:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:15Z" name: ensemble-graph-091ad-8999dbf5b-s5rpc.18a6ea477096c8bb namespace: kserve-ci-e2e-test resourceVersion: "19600" uid: 7303ecda-6759-476e-be45-9e38f72b2bc1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-091ad} kind: Pod name: ensemble-graph-091ad-8999dbf5b-s5rpc namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: 003dfaa6-16d1-4f32-85d6-5d114a54c1fa kind: Event lastTimestamp: "2026-04-16T18:38:15Z" message: 'Created container: ensemble-graph-091ad' metadata: creationTimestamp: "2026-04-16T18:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:15Z" name: ensemble-graph-091ad-8999dbf5b-s5rpc.18a6ea477331d3ba namespace: kserve-ci-e2e-test resourceVersion: "19601" uid: b668336b-5e79-49c6-8e62-63480e94ea12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-091ad} kind: Pod name: ensemble-graph-091ad-8999dbf5b-s5rpc namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: 003dfaa6-16d1-4f32-85d6-5d114a54c1fa kind: Event lastTimestamp: "2026-04-16T18:38:15Z" message: Started container ensemble-graph-091ad metadata: creationTimestamp: "2026-04-16T18:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:15Z" name: ensemble-graph-091ad-8999dbf5b-s5rpc.18a6ea4773a78e5f namespace: kserve-ci-e2e-test resourceVersion: "19602" uid: fbdcc3a6-67a6-4f2d-89a6-9c3f91b0796f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-091ad} kind: Pod name: ensemble-graph-091ad-8999dbf5b-s5rpc namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: 003dfaa6-16d1-4f32-85d6-5d114a54c1fa kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Stopping container ensemble-graph-091ad metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:25Z" name: ensemble-graph-091ad-8999dbf5b-s5rpc.18a6ea49ad39a45f namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: ed3facba-5885-4498-b41a-2026dc64b0b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:38:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-091ad} kind: Pod name: ensemble-graph-091ad-8999dbf5b-s5rpc namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: 003dfaa6-16d1-4f32-85d6-5d114a54c1fa kind: Event lastTimestamp: "2026-04-16T18:38:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:38:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:52Z" name: ensemble-graph-091ad-8999dbf5b-s5rpc.18a6ea4a2ad4e6a5 namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: d0f2edbf-1edb-4d67-8e22-4a323c2706b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-091ad-8999dbf5b namespace: kserve-ci-e2e-test resourceVersion: "19577" uid: d4673451-5d51-47b3-94a6-11d8575aa330 kind: Event lastTimestamp: "2026-04-16T18:38:15Z" message: 'Created pod: ensemble-graph-091ad-8999dbf5b-s5rpc' metadata: creationTimestamp: "2026-04-16T18:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:15Z" name: ensemble-graph-091ad-8999dbf5b.18a6ea4755ae3eb1 namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: b0b4f648-d7c6-484d-a9e8-81ee8d40798d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-091ad namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: 90539908-5e69-45eb-8179-2872acf51dab kind: Event lastTimestamp: "2026-04-16T18:38:15Z" message: Scaled up replica set ensemble-graph-091ad-8999dbf5b from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:15Z" name: ensemble-graph-091ad.18a6ea4754e396b0 namespace: kserve-ci-e2e-test resourceVersion: "19578" uid: f7cc6387-18ba-48e6-9113-11d8a963f01b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-091ad namespace: kserve-ci-e2e-test resourceVersion: "19591" uid: 2421c371-7233-48a4-b32e-6215affa441e kind: Event lastTimestamp: "2026-04-16T18:38:15Z" message: InferenceGraph [ensemble-graph-091ad] is Ready metadata: creationTimestamp: "2026-04-16T18:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:15Z" name: ensemble-graph-091ad.18a6ea475ac118c9 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 6be54794-7430-4393-b910-12df733423ea reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:39:35.433327Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-976d9-5dd8566c9d-lqx8z namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: b73e10f9-f7c9-4b71-8157-dcf5e6120e91 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-976d9-5dd8566c9d-lqx8z to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:39:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:39:35Z" name: ensemble-graph-976d9-5dd8566c9d-lqx8z.18a6ea5a07386b18 namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: 7f62d184-7e8d-4209-9c6d-56368987642e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:35Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-976d9-5dd8566c9d-lqx8z namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: b73e10f9-f7c9-4b71-8157-dcf5e6120e91 kind: Event lastTimestamp: "2026-04-16T18:39:35Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:39:35Z" name: ensemble-graph-976d9-5dd8566c9d-lqx8z.18a6ea5a1ff50674 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: a9e431b8-1f79-4ff3-b10a-47adf8886dfd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-976d9} kind: Pod name: ensemble-graph-976d9-5dd8566c9d-lqx8z namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: b73e10f9-f7c9-4b71-8157-dcf5e6120e91 kind: Event lastTimestamp: "2026-04-16T18:39:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:35Z" name: ensemble-graph-976d9-5dd8566c9d-lqx8z.18a6ea5a21416582 namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 1b9fd7f4-2c24-467c-a8f4-e9e810ad6f9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-976d9} kind: Pod name: ensemble-graph-976d9-5dd8566c9d-lqx8z namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: b73e10f9-f7c9-4b71-8157-dcf5e6120e91 kind: Event lastTimestamp: "2026-04-16T18:39:35Z" message: 'Created container: ensemble-graph-976d9' metadata: creationTimestamp: "2026-04-16T18:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:35Z" name: ensemble-graph-976d9-5dd8566c9d-lqx8z.18a6ea5a23c3fdc2 namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: 3938aff2-d2f1-461f-9b5c-204e0d755a79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-976d9} kind: Pod name: ensemble-graph-976d9-5dd8566c9d-lqx8z namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: b73e10f9-f7c9-4b71-8157-dcf5e6120e91 kind: Event lastTimestamp: "2026-04-16T18:39:35Z" message: Started container ensemble-graph-976d9 metadata: creationTimestamp: "2026-04-16T18:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:35Z" name: ensemble-graph-976d9-5dd8566c9d-lqx8z.18a6ea5a2443d35b namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: cb64952a-2999-4a67-bda6-93d76edfb38c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-976d9} kind: Pod name: ensemble-graph-976d9-5dd8566c9d-lqx8z namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: b73e10f9-f7c9-4b71-8157-dcf5e6120e91 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Stopping container ensemble-graph-976d9 metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:50Z" name: ensemble-graph-976d9-5dd8566c9d-lqx8z.18a6eacd33a89429 namespace: kserve-ci-e2e-test resourceVersion: "24103" uid: 16a9b271-b975-4066-b7b2-2781ce4f1918 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:47:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-976d9} kind: Pod name: ensemble-graph-976d9-5dd8566c9d-lqx8z namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: b73e10f9-f7c9-4b71-8157-dcf5e6120e91 kind: Event lastTimestamp: "2026-04-16T18:48:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:47:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:17Z" name: ensemble-graph-976d9-5dd8566c9d-lqx8z.18a6eacdc54b5e4c namespace: kserve-ci-e2e-test resourceVersion: "24419" uid: 34db3edc-0979-491a-b6f9-cf05e5f266df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-976d9-5dd8566c9d namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 694210b7-f208-42e2-8ebf-75329133c939 kind: Event lastTimestamp: "2026-04-16T18:39:35Z" message: 'Created pod: ensemble-graph-976d9-5dd8566c9d-lqx8z' metadata: creationTimestamp: "2026-04-16T18:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:35Z" name: ensemble-graph-976d9-5dd8566c9d.18a6ea5a069b4db5 namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: aea80339-8b85-4622-8b2e-bab894c193f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-976d9 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: f3a165d8-c400-4a2c-ad7e-13706c619e85 kind: Event lastTimestamp: "2026-04-16T18:39:35Z" message: Scaled up replica set ensemble-graph-976d9-5dd8566c9d from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:35Z" name: ensemble-graph-976d9.18a6ea5a05b91509 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 2a07c98f-d1dc-4868-8d62-92df35129dd4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-976d9 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 32edb19c-a186-4f85-82c4-bd74c0d73a2c kind: Event lastTimestamp: "2026-04-16T18:39:35Z" message: InferenceGraph [ensemble-graph-976d9] is Ready metadata: creationTimestamp: "2026-04-16T18:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:35Z" name: ensemble-graph-976d9.18a6ea5a0993070e namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 3983ff2e-505a-467d-9446-b5969f2e41c9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:39:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-976d9 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: adc375d7-44a6-45eb-ba76-3498db483b7a kind: Event lastTimestamp: "2026-04-16T18:40:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:20Z" name: ensemble-graph-976d9.18a6ea5d8457114e namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 9c95534b-1832-40e6-9271-d2bdb04e26cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:39:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-976d9 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: adc375d7-44a6-45eb-ba76-3498db483b7a kind: Event lastTimestamp: "2026-04-16T18:40:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:20Z" name: ensemble-graph-976d9.18a6ea5d84e90687 namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 6ee70b78-bfd2-4854-a847-5641f53342c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:37:05.083958Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2 namespace: kserve-ci-e2e-test resourceVersion: "18847" uid: 0526f50e-2db5-427f-854b-5e708f881c7d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-091ad-predictor-8bbcb8548-l26p2 to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2.18a6ea3705b320d8 namespace: kserve-ci-e2e-test resourceVersion: "18852" uid: 4a33d40d-d4b5-4027-9a19-7cd9136ba7b6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2 namespace: kserve-ci-e2e-test resourceVersion: "18856" uid: 0526f50e-2db5-427f-854b-5e708f881c7d kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2.18a6ea370dcc02a9 namespace: kserve-ci-e2e-test resourceVersion: "18864" uid: ec39d535-82c8-4ce5-9616-4664b1a31993 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 0526f50e-2db5-427f-854b-5e708f881c7d kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2.18a6ea370f55003a namespace: kserve-ci-e2e-test resourceVersion: "18867" uid: 9f9a1233-d0c4-44d6-9cae-17d3a8d6a39e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 0526f50e-2db5-427f-854b-5e708f881c7d kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2.18a6ea37152555ef namespace: kserve-ci-e2e-test resourceVersion: "18872" uid: 70480d4e-b4f3-4365-b28a-1ce22803da90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 0526f50e-2db5-427f-854b-5e708f881c7d kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2.18a6ea37159844e1 namespace: kserve-ci-e2e-test resourceVersion: "18873" uid: 943025e7-6106-4061-9547-e6d832acbe8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 0526f50e-2db5-427f-854b-5e708f881c7d kind: Event lastTimestamp: "2026-04-16T18:38:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:26Z" name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2.18a6ea37350bf985 namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: efa9de41-f2b6-475c-8105-bf93cc543125 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 0526f50e-2db5-427f-854b-5e708f881c7d kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-091ad-predictor-8bbcb8548-l26p2.18a6ea49b3a36748 namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 06282f71-aaee-4205-b9ca-f1d12795b79e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-091ad-predictor-8bbcb8548 namespace: kserve-ci-e2e-test resourceVersion: "18835" uid: 4fa6cef1-b514-4808-99ec-32c0edacdb2b kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: 'Created pod: error-404-isvc-091ad-predictor-8bbcb8548-l26p2' metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-091ad-predictor-8bbcb8548.18a6ea370533d97a namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: 9a4095d4-7cfd-49aa-924c-156bd5689443 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "18827" uid: 275a7ea5-0ba0-4426-bc06-8a7b62e7a51c kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: Scaled up replica set error-404-isvc-091ad-predictor-8bbcb8548 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-091ad-predictor.18a6ea3703baaf86 namespace: kserve-ci-e2e-test resourceVersion: "18838" uid: 23f0e601-acd2-4f98-aae7-44a539bdeeb3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:37:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 44b15b1e-832e-4bc0-837d-4079ef7865a4 kind: Event lastTimestamp: "2026-04-16T18:37:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:50Z" name: error-404-isvc-091ad-predictor.18a6ea3a82ed26e8 namespace: kserve-ci-e2e-test resourceVersion: "19221" uid: 9d2a4e15-0dc2-4817-a0e7-21c82c80716c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:37:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 44b15b1e-832e-4bc0-837d-4079ef7865a4 kind: Event lastTimestamp: "2026-04-16T18:37:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:50Z" name: error-404-isvc-091ad-predictor.18a6ea3a83628c13 namespace: kserve-ci-e2e-test resourceVersion: "19222" uid: 500b62a0-e199-4c0b-bdc4-d9032ce91e7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19001" uid: 44b15b1e-832e-4bc0-837d-4079ef7865a4 kind: Event lastTimestamp: "2026-04-16T18:38:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:20Z" name: error-404-isvc-091ad-predictor.18a6ea45000ad9b2 namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: a9f6285a-ba97-4a2d-b168-0b8d9068dffd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "19001" uid: 44b15b1e-832e-4bc0-837d-4079ef7865a4 kind: Event lastTimestamp: "2026-04-16T18:38:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:20Z" name: error-404-isvc-091ad-predictor.18a6ea45006e4e80 namespace: kserve-ci-e2e-test resourceVersion: "19644" uid: b1fce861-f093-4c6e-b228-c3e0d9fe6d52 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-091ad namespace: kserve-ci-e2e-test resourceVersion: "18809" uid: 1d17bc32-f099-42f6-a679-1a469f71d23c kind: Event lastTimestamp: "2026-04-16T18:37:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:56Z" name: error-404-isvc-091ad.18a6ea3703c09de4 namespace: kserve-ci-e2e-test resourceVersion: "19411" uid: 81309b0f-7d6a-46f9-bb83-086f79fc50f8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-091ad namespace: kserve-ci-e2e-test resourceVersion: "18809" uid: 1d17bc32-f099-42f6-a679-1a469f71d23c kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: 'Failed to update status for InferenceService "error-404-isvc-091ad": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-091ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-091ad.18a6ea3704514581 namespace: kserve-ci-e2e-test resourceVersion: "18844" uid: 98a72aae-042f-4bc7-bf72-8ff6e17dd4bd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-091ad namespace: kserve-ci-e2e-test resourceVersion: "18809" uid: 1d17bc32-f099-42f6-a679-1a469f71d23c kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-091ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-091ad.18a6ea370451757e namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: fed4abe0-4f05-4ef0-872a-5c4f13f9005c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-091ad namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 1d17bc32-f099-42f6-a679-1a469f71d23c kind: Event lastTimestamp: "2026-04-16T18:37:56Z" message: InferenceService [error-404-isvc-091ad] is Ready metadata: creationTimestamp: "2026-04-16T18:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:56Z" name: error-404-isvc-091ad.18a6ea431a22dec2 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: 7618953f-62d0-45b3-9d76-a837341d913e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:37:54.786933Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8 namespace: kserve-ci-e2e-test resourceVersion: "19344" uid: 86f26ab8-57ab-4d69-a4b6-452207b4d41f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8 to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:37:54Z" name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8.18a6ea42983a5f33 namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: a2cb5c7d-ea6e-484a-ad83-a800cf4b1adb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8 namespace: kserve-ci-e2e-test resourceVersion: "19354" uid: 86f26ab8-57ab-4d69-a4b6-452207b4d41f kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:37:54Z" name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8.18a6ea42a0b87a28 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 1dc7fa78-2a75-49e9-a607-fd6499c649e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8 namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 86f26ab8-57ab-4d69-a4b6-452207b4d41f kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:54Z" name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8.18a6ea42a1fddd8e namespace: kserve-ci-e2e-test resourceVersion: "19370" uid: 6bf32de8-d837-4e7e-8cd6-71b14549b825 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8 namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 86f26ab8-57ab-4d69-a4b6-452207b4d41f kind: Event lastTimestamp: "2026-04-16T18:37:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:55Z" name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8.18a6ea42a75c0c63 namespace: kserve-ci-e2e-test resourceVersion: "19373" uid: 83950bf3-94de-495f-8352-4f1c375f19be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8 namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 86f26ab8-57ab-4d69-a4b6-452207b4d41f kind: Event lastTimestamp: "2026-04-16T18:37:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:55Z" name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8.18a6ea42a7d71a9e namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: d8f48f40-dbea-4c96-8a5c-ad857e5e8e66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8 namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 86f26ab8-57ab-4d69-a4b6-452207b4d41f kind: Event lastTimestamp: "2026-04-16T18:39:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:17Z" name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8.18a6ea42e3b8a1be namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 158a7c12-1e64-40ab-849a-ce82ebf4eaca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8 namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 86f26ab8-57ab-4d69-a4b6-452207b4d41f kind: Event lastTimestamp: "2026-04-16T18:39:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:14Z" name: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8.18a6ea553ebb6b95 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 47f92b5c-ad54-4713-8ebc-52463e4a2fb1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0f5d9-predictor-67c84f489d namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: 5b025564-5549-477e-8948-8dfd1ffc083d kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: 'Created pod: error-404-isvc-0f5d9-predictor-67c84f489d-xh7g8' metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:54Z" name: error-404-isvc-0f5d9-predictor-67c84f489d.18a6ea4297cc632f namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: a1d640ec-9e95-42c3-96a6-f2ff0c2d708d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19326" uid: ed3b95d5-c561-439e-830f-ed15c4ec071a kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Scaled up replica set error-404-isvc-0f5d9-predictor-67c84f489d from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:54Z" name: error-404-isvc-0f5d9-predictor.18a6ea42960ef2f7 namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: 7f5ef13b-e84a-4d9d-b266-ede8ffbdff44 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19332" uid: f242730c-5c1d-47ed-a810-341102277732 kind: Event lastTimestamp: "2026-04-16T18:38:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:09Z" name: error-404-isvc-0f5d9-predictor.18a6ea4614d1f907 namespace: kserve-ci-e2e-test resourceVersion: "19524" uid: 9b956da7-c091-43df-800c-72364e41dc7f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19332" uid: f242730c-5c1d-47ed-a810-341102277732 kind: Event lastTimestamp: "2026-04-16T18:38:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:09Z" name: error-404-isvc-0f5d9-predictor.18a6ea4615461ed0 namespace: kserve-ci-e2e-test resourceVersion: "19526" uid: ae43df59-00e7-40d7-b934-009c5d16ff0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19525" uid: f242730c-5c1d-47ed-a810-341102277732 kind: Event lastTimestamp: "2026-04-16T18:39:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:09Z" name: error-404-isvc-0f5d9-predictor.18a6ea49941f6114 namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: 7cd2f005-5d8b-46d5-966b-dabdf04accf4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19525" uid: f242730c-5c1d-47ed-a810-341102277732 kind: Event lastTimestamp: "2026-04-16T18:39:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:09Z" name: error-404-isvc-0f5d9-predictor.18a6ea49949bee89 namespace: kserve-ci-e2e-test resourceVersion: "20230" uid: 4f1d9bc6-7039-4e77-a1d4-20477b33b651 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 5a3e794a-f2d5-443b-a633-e0cbba75864c kind: Event lastTimestamp: "2026-04-16T18:38:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:47Z" name: error-404-isvc-0f5d9.18a6ea42960fd941 namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: e58201f9-e84e-488a-b6aa-627355f1faf4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 5a3e794a-f2d5-443b-a633-e0cbba75864c kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: 'Failed to update status for InferenceService "error-404-isvc-0f5d9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0f5d9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:54Z" name: error-404-isvc-0f5d9.18a6ea4296ba2128 namespace: kserve-ci-e2e-test resourceVersion: "19340" uid: ec386bd8-1385-4559-ac8a-f83054814976 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 5a3e794a-f2d5-443b-a633-e0cbba75864c kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0f5d9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:54Z" name: error-404-isvc-0f5d9.18a6ea4296ba5a51 namespace: kserve-ci-e2e-test resourceVersion: "19341" uid: b50b4741-724f-4c63-97fe-3995b26bd8c9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "20011" uid: 5a3e794a-f2d5-443b-a633-e0cbba75864c kind: Event lastTimestamp: "2026-04-16T18:38:47Z" message: InferenceService [error-404-isvc-0f5d9] is Ready metadata: creationTimestamp: "2026-04-16T18:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:47Z" name: error-404-isvc-0f5d9.18a6ea4ec8baf07a namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 53dadd64-5815-4e29-be73-2deaa59790cc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:26:09.953510Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: dd291430-1c3f-47fa-bd8c-99941ebf9d84 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3596a-predictor-79bf5f4878-g9cdk to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:26:09Z" name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk.18a6e99e7ce1cd56 namespace: kserve-ci-e2e-test resourceVersion: "13378" uid: c70a73f9-1c8f-44c2-99c9-41f87ffe80c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk namespace: kserve-ci-e2e-test resourceVersion: "13383" uid: dd291430-1c3f-47fa-bd8c-99941ebf9d84 kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:26:10Z" name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk.18a6e99e8625fdd0 namespace: kserve-ci-e2e-test resourceVersion: "13409" uid: b007f33f-6024-4322-970f-699e8ab95fad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk namespace: kserve-ci-e2e-test resourceVersion: "13377" uid: dd291430-1c3f-47fa-bd8c-99941ebf9d84 kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:10Z" name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk.18a6e99e87b7239e namespace: kserve-ci-e2e-test resourceVersion: "13411" uid: 3950b190-9a46-41f7-944f-dffd8c75b20a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk namespace: kserve-ci-e2e-test resourceVersion: "13377" uid: dd291430-1c3f-47fa-bd8c-99941ebf9d84 kind: Event lastTimestamp: "2026-04-16T18:26:22Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" in 11.965s (11.965s including waiting). Image size: 1333165160 bytes.' metadata: creationTimestamp: "2026-04-16T18:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:22Z" name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk.18a6e9a150e3e74a namespace: kserve-ci-e2e-test resourceVersion: "13581" uid: 27abecde-7a89-4a6f-9232-6928ad053049 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk namespace: kserve-ci-e2e-test resourceVersion: "13377" uid: dd291430-1c3f-47fa-bd8c-99941ebf9d84 kind: Event lastTimestamp: "2026-04-16T18:26:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:22Z" name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk.18a6e9a1593dcd99 namespace: kserve-ci-e2e-test resourceVersion: "13583" uid: 7768b2d2-53c3-4d05-8b5a-e4591c396c23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk namespace: kserve-ci-e2e-test resourceVersion: "13377" uid: dd291430-1c3f-47fa-bd8c-99941ebf9d84 kind: Event lastTimestamp: "2026-04-16T18:26:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:22Z" name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk.18a6e9a159b6f0fd namespace: kserve-ci-e2e-test resourceVersion: "13584" uid: 54b5acf1-d186-4e04-b5e0-ffea37027ac6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk namespace: kserve-ci-e2e-test resourceVersion: "13377" uid: dd291430-1c3f-47fa-bd8c-99941ebf9d84 kind: Event lastTimestamp: "2026-04-16T18:27:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:03Z" name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk.18a6e9a17225503b namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: 43362ce3-b82c-45bc-91c7-72a059aed139 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk namespace: kserve-ci-e2e-test resourceVersion: "13377" uid: dd291430-1c3f-47fa-bd8c-99941ebf9d84 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk.18a6e9b37af6fe04 namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: 7d56b55e-5d52-45a2-8f89-10ff6eb35607 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3596a-predictor-79bf5f4878 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 79f53a93-82a9-4d13-90de-5bdcb52712d9 kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: 'Created pod: error-404-isvc-3596a-predictor-79bf5f4878-g9cdk' metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:09Z" name: error-404-isvc-3596a-predictor-79bf5f4878.18a6e99e7c060ac2 namespace: kserve-ci-e2e-test resourceVersion: "13375" uid: 2c5607d6-c603-4641-9bf1-e0134c43b133 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13364" uid: 20d8e4ea-2a50-4898-b756-e5c0c55b2a5a kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: Scaled up replica set error-404-isvc-3596a-predictor-79bf5f4878 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:09Z" name: error-404-isvc-3596a-predictor.18a6e99e7a669879 namespace: kserve-ci-e2e-test resourceVersion: "13369" uid: 0d9d3fb3-2bbe-4f65-993c-b96d67753cbf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13366" uid: c95396d6-99c8-4d2d-821c-840722d5f7af kind: Event lastTimestamp: "2026-04-16T18:26:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:39Z" name: error-404-isvc-3596a-predictor.18a6e9a1f91a5c8a namespace: kserve-ci-e2e-test resourceVersion: "13764" uid: 17d5613a-653a-4323-a065-0e956b46a712 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13366" uid: c95396d6-99c8-4d2d-821c-840722d5f7af kind: Event lastTimestamp: "2026-04-16T18:26:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:39Z" name: error-404-isvc-3596a-predictor.18a6e9a1f995dbdc namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: 8e0cede8-4592-495b-b3b6-91e582d3240d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:26:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13633" uid: c95396d6-99c8-4d2d-821c-840722d5f7af kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:26:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-3596a-predictor.18a6e9a8f73d6e51 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 937267b3-66d1-4d9c-87f9-f04b2ec12d6b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:26:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13633" uid: c95396d6-99c8-4d2d-821c-840722d5f7af kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:26:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-3596a-predictor.18a6e9a8f7afe364 namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: 7f99cd18-4534-4dac-88cd-5355f2471906 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3596a namespace: kserve-ci-e2e-test resourceVersion: "13349" uid: 4dbc2c63-2ee5-436d-b74e-bbff539ee74a kind: Event lastTimestamp: "2026-04-16T18:27:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:13Z" name: error-404-isvc-3596a.18a6e99e7a3e5738 namespace: kserve-ci-e2e-test resourceVersion: "14056" uid: e2b17460-4ca7-40bb-89b6-50269b26ef3d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3596a namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: 4dbc2c63-2ee5-436d-b74e-bbff539ee74a kind: Event lastTimestamp: "2026-04-16T18:27:13Z" message: InferenceService [error-404-isvc-3596a] is Ready metadata: creationTimestamp: "2026-04-16T18:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:13Z" name: error-404-isvc-3596a.18a6e9ad574e76d2 namespace: kserve-ci-e2e-test resourceVersion: "14055" uid: 1e8c1e0d-1e87-4ccc-9687-e07a73d764fa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:49:10.641775Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7 namespace: kserve-ci-e2e-test resourceVersion: "25103" uid: 392b7c5d-9a36-4cd4-a37f-e03215fff699 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-56ec1-predictor-f476465cd-bqvv7 to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7.18a6eadff450dbce namespace: kserve-ci-e2e-test resourceVersion: "25108" uid: edbd8991-a06e-4d96-ac8f-470f75c42f89 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7 namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 392b7c5d-9a36-4cd4-a37f-e03215fff699 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7.18a6eadffc34f0f5 namespace: kserve-ci-e2e-test resourceVersion: "25122" uid: c729387c-d100-4d08-9f84-c6762eefbe75 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7 namespace: kserve-ci-e2e-test resourceVersion: "25105" uid: 392b7c5d-9a36-4cd4-a37f-e03215fff699 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7.18a6eadffd892417 namespace: kserve-ci-e2e-test resourceVersion: "25126" uid: 290dd43e-627c-4175-823a-39832248364d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7 namespace: kserve-ci-e2e-test resourceVersion: "25105" uid: 392b7c5d-9a36-4cd4-a37f-e03215fff699 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7.18a6eae00126a73c namespace: kserve-ci-e2e-test resourceVersion: "25131" uid: 2f04f4a3-b334-4d1a-8e62-24994fc4fc80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7 namespace: kserve-ci-e2e-test resourceVersion: "25105" uid: 392b7c5d-9a36-4cd4-a37f-e03215fff699 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7.18a6eae001870fe8 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 2a7fc804-45c5-4c88-98d8-09c4d1882b85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7 namespace: kserve-ci-e2e-test resourceVersion: "25105" uid: 392b7c5d-9a36-4cd4-a37f-e03215fff699 kind: Event lastTimestamp: "2026-04-16T18:49:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:52Z" name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7.18a6eae022c1a945 namespace: kserve-ci-e2e-test resourceVersion: "25535" uid: c6c5df9c-c1e9-441f-88c4-6293520da818 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:58:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7 namespace: kserve-ci-e2e-test resourceVersion: "25105" uid: 392b7c5d-9a36-4cd4-a37f-e03215fff699 kind: Event lastTimestamp: "2026-04-16T18:58:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:58:35Z" name: error-404-isvc-56ec1-predictor-f476465cd-bqvv7.18a6eb6375fca35d namespace: kserve-ci-e2e-test resourceVersion: "29559" uid: 43bc1f53-0834-4e82-956c-2a0a1119be74 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-56ec1-predictor-f476465cd namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 3a11f4f8-6bfc-4f71-a697-e786194fdbb2 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: 'Created pod: error-404-isvc-56ec1-predictor-f476465cd-bqvv7' metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-56ec1-predictor-f476465cd.18a6eadff3d75846 namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: 9570b2f2-6582-4bb9-afb5-65002bff0d57 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 9421cd3b-baa6-4b88-b5eb-8ffdbe4d7322 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Scaled up replica set error-404-isvc-56ec1-predictor-f476465cd from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-56ec1-predictor.18a6eadff240eb68 namespace: kserve-ci-e2e-test resourceVersion: "25097" uid: 299dbeda-ed46-41f9-a3c9-e89ab4066c34 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25090" uid: d8a677b4-22a0-4e66-bec7-b30bfd026fc4 kind: Event lastTimestamp: "2026-04-16T18:49:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:55Z" name: error-404-isvc-56ec1-predictor.18a6eae370ba0550 namespace: kserve-ci-e2e-test resourceVersion: "25567" uid: 0ef15239-f6dd-47bd-b952-02767fbe9371 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25090" uid: d8a677b4-22a0-4e66-bec7-b30bfd026fc4 kind: Event lastTimestamp: "2026-04-16T18:49:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:55Z" name: error-404-isvc-56ec1-predictor.18a6eae3712ec515 namespace: kserve-ci-e2e-test resourceVersion: "25568" uid: a28aa67b-c4a2-4a23-8b2c-115d4f6dff3e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:50:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: d8a677b4-22a0-4e66-bec7-b30bfd026fc4 kind: Event lastTimestamp: "2026-04-16T18:50:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:25Z" name: error-404-isvc-56ec1-predictor.18a6eaededbc2dd8 namespace: kserve-ci-e2e-test resourceVersion: "26132" uid: dd1a51fb-7c9d-4c7c-8bf4-3591fdeca7cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:50:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: d8a677b4-22a0-4e66-bec7-b30bfd026fc4 kind: Event lastTimestamp: "2026-04-16T18:50:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:25Z" name: error-404-isvc-56ec1-predictor.18a6eaedee474924 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: a6523420-96f5-4e83-9cf6-9fda1fa206c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25073" uid: 1dc57277-f858-4fc3-84e4-d5d2c0f6e146 kind: Event lastTimestamp: "2026-04-16T18:50:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:50:02Z" name: error-404-isvc-56ec1.18a6eadff2158a3b namespace: kserve-ci-e2e-test resourceVersion: "25636" uid: af4cdf18-e654-410f-a50d-5fed4c032e0b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25073" uid: 1dc57277-f858-4fc3-84e4-d5d2c0f6e146 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-56ec1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-56ec1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-56ec1.18a6eadff2932b2d namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: 5618bcf8-222e-4869-85b5-d3ea9177ea86 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25073" uid: 1dc57277-f858-4fc3-84e4-d5d2c0f6e146 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-56ec1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-56ec1.18a6eadff2937289 namespace: kserve-ci-e2e-test resourceVersion: "25101" uid: 3bbd837a-7a1b-4818-86f5-614410121f9d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25633" uid: 1dc57277-f858-4fc3-84e4-d5d2c0f6e146 kind: Event lastTimestamp: "2026-04-16T18:50:02Z" message: InferenceService [error-404-isvc-56ec1] is Ready metadata: creationTimestamp: "2026-04-16T18:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:50:02Z" name: error-404-isvc-56ec1.18a6eaec06e2918b namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 814622be-7ebf-49a8-9c78-9cc969f46e40 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:39:15.038095Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 82e0022e-89ce-4a30-9ce0-6794d78d213d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:39:15Z" name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb.18a6ea554791e47e namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: c39f20e8-a4ab-4bd1-818d-bebff97013e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: 82e0022e-89ce-4a30-9ce0-6794d78d213d kind: Event lastTimestamp: "2026-04-16T18:39:15Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:39:15Z" name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb.18a6ea554f0e11cd namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 6e67b940-ac5a-4a5f-8ec8-a3156c5581cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 82e0022e-89ce-4a30-9ce0-6794d78d213d kind: Event lastTimestamp: "2026-04-16T18:39:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:15Z" name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb.18a6ea55508acd8c namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 9e3a9534-d6ae-4de8-91e8-7d92fcb82462 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 82e0022e-89ce-4a30-9ce0-6794d78d213d kind: Event lastTimestamp: "2026-04-16T18:39:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:15Z" name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb.18a6ea5553a7048b namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 69251ee9-9052-4393-8f06-4fe5bad83eb3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 82e0022e-89ce-4a30-9ce0-6794d78d213d kind: Event lastTimestamp: "2026-04-16T18:39:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:15Z" name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb.18a6ea555418ca62 namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: b6afb35d-421b-41e9-86be-546c14df16c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 82e0022e-89ce-4a30-9ce0-6794d78d213d kind: Event lastTimestamp: "2026-04-16T18:39:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:56Z" name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb.18a6ea5559e6363a namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: cad1a55d-f645-4204-a8d9-59a2c9685756 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 82e0022e-89ce-4a30-9ce0-6794d78d213d kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:39Z" name: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb.18a6ead8c1bf5002 namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: 837bb313-04e8-477f-b47c-fb2095f376f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5aeb1-predictor-6455bd6978 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: e5a56187-9d90-47c1-9abe-e8a3051ffd23 kind: Event lastTimestamp: "2026-04-16T18:39:15Z" message: 'Created pod: error-404-isvc-5aeb1-predictor-6455bd6978-5jrrb' metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:15Z" name: error-404-isvc-5aeb1-predictor-6455bd6978.18a6ea5546d48161 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 7147c94b-dcdd-47ab-92d1-9b38393ef148 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: d18d6107-7d9b-423a-8009-0a6644ae0d1d kind: Event lastTimestamp: "2026-04-16T18:39:14Z" message: Scaled up replica set error-404-isvc-5aeb1-predictor-6455bd6978 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:14Z" name: error-404-isvc-5aeb1-predictor.18a6ea5544cca484 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: 8788c7aa-460b-4feb-b7c2-c8a3ccadeac8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 349a5369-6a1e-41ee-ae89-f8537c251ac1 kind: Event lastTimestamp: "2026-04-16T18:39:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:30Z" name: error-404-isvc-5aeb1-predictor.18a6ea58c3645af0 namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: edd52122-2843-4b14-a593-8bf988cbf3f0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 349a5369-6a1e-41ee-ae89-f8537c251ac1 kind: Event lastTimestamp: "2026-04-16T18:39:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:30Z" name: error-404-isvc-5aeb1-predictor.18a6ea58c3d7bcbe namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 511a64ad-c51e-4039-a81d-9e88e7c818ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:39:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 349a5369-6a1e-41ee-ae89-f8537c251ac1 kind: Event lastTimestamp: "2026-04-16T18:40:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:30Z" name: error-404-isvc-5aeb1-predictor.18a6ea5c42f8b260 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: 98693c21-a427-4ef2-be83-ad337ac84a7e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:39:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 349a5369-6a1e-41ee-ae89-f8537c251ac1 kind: Event lastTimestamp: "2026-04-16T18:40:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:30Z" name: error-404-isvc-5aeb1-predictor.18a6ea5c4375ab49 namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: d3e61e8d-7800-434e-9cfd-3ef3043d6299 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: b4787b0e-68f6-460e-844a-d07160f3e394 kind: Event lastTimestamp: "2026-04-16T18:40:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:06Z" name: error-404-isvc-5aeb1.18a6ea5544a48f52 namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 9a57d6b5-741e-4cbd-b1e5-2d75e3eddae6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: b4787b0e-68f6-460e-844a-d07160f3e394 kind: Event lastTimestamp: "2026-04-16T18:39:14Z" message: 'Failed to update status for InferenceService "error-404-isvc-5aeb1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5aeb1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:14Z" name: error-404-isvc-5aeb1.18a6ea554514b927 namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: be2085d7-18b7-49eb-a454-39fa59dbc320 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: b4787b0e-68f6-460e-844a-d07160f3e394 kind: Event lastTimestamp: "2026-04-16T18:39:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5aeb1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:15Z" name: error-404-isvc-5aeb1.18a6ea554514f796 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 461aec23-9d81-4947-aba4-bf0bae3cda81 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: b4787b0e-68f6-460e-844a-d07160f3e394 kind: Event lastTimestamp: "2026-04-16T18:40:06Z" message: InferenceService [error-404-isvc-5aeb1] is Ready metadata: creationTimestamp: "2026-04-16T18:40:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:06Z" name: error-404-isvc-5aeb1.18a6ea613ebb9bdf namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: b4fe2330-1d25-4ce2-83de-8177010e2e80 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:27:40.162661Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: befbd30e-91ed-4fac-a5aa-111f3b56f22c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6138a-predictor-64d77c84bd-gcvtv to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv.18a6e9b37dc43c49 namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 0f667512-2e3e-4471-8aa7-168b7230cc48 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: befbd30e-91ed-4fac-a5aa-111f3b56f22c kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv.18a6e9b386577944 namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: d2c4d218-a39a-4769-8737-705dbebedca9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: befbd30e-91ed-4fac-a5aa-111f3b56f22c kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv.18a6e9b387b33fbe namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 4251190b-f8e8-4ed9-b443-82973500f45d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: befbd30e-91ed-4fac-a5aa-111f3b56f22c kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv.18a6e9b38e610dda namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: 9e8eee9b-3c51-4010-98b8-d5fbe9759ad2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: befbd30e-91ed-4fac-a5aa-111f3b56f22c kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv.18a6e9b38ee250e1 namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: 23503808-50fd-4cce-a999-998fd7a8b3d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:27:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: befbd30e-91ed-4fac-a5aa-111f3b56f22c kind: Event lastTimestamp: "2026-04-16T18:28:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:22Z" name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv.18a6e9b3e7a0f6a9 namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 4aa1b821-61bf-4131-a6b7-7820caf1e28c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: befbd30e-91ed-4fac-a5aa-111f3b56f22c kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:05Z" name: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv.18a6ea3704d86199 namespace: kserve-ci-e2e-test resourceVersion: "18848" uid: 7ff4b505-dd3a-4953-bf10-f1d8ef9eff6d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6138a-predictor-64d77c84bd namespace: kserve-ci-e2e-test resourceVersion: "14401" uid: d62cda8d-c18b-4ff4-bfbf-bbb00301592e kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'Created pod: error-404-isvc-6138a-predictor-64d77c84bd-gcvtv' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-6138a-predictor-64d77c84bd.18a6e9b37d649bf6 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 42724d5e-092e-4e9a-9e69-f4ee6a5d9283 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: 194f45b7-9113-425c-9159-f6205d3786f4 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Scaled up replica set error-404-isvc-6138a-predictor-64d77c84bd from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-6138a-predictor.18a6e9b37c129015 namespace: kserve-ci-e2e-test resourceVersion: "14403" uid: 996b0f2b-aaad-48d2-888a-bffc61351f27 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: 3e5fc614-7d4c-4aaa-be8e-9809e31087c2 kind: Event lastTimestamp: "2026-04-16T18:28:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:10Z" name: error-404-isvc-6138a-predictor.18a6e9b6fab37148 namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: 023d1121-e81c-454a-bc2d-354d66bc371b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: 3e5fc614-7d4c-4aaa-be8e-9809e31087c2 kind: Event lastTimestamp: "2026-04-16T18:28:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:10Z" name: error-404-isvc-6138a-predictor.18a6e9b6fb254d92 namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: 39ffaef6-71b4-4ff9-b5d6-2c48f2352012 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3e5fc614-7d4c-4aaa-be8e-9809e31087c2 kind: Event lastTimestamp: "2026-04-16T18:28:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:55Z" name: error-404-isvc-6138a-predictor.18a6e9bdf98540e3 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: d18da12c-3ae8-4a6d-985c-4ad987668a2c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3e5fc614-7d4c-4aaa-be8e-9809e31087c2 kind: Event lastTimestamp: "2026-04-16T18:28:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:55Z" name: error-404-isvc-6138a-predictor.18a6e9bdfa4da323 namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 5eaed6d5-e057-4f2e-961e-011761fc48d8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6138a namespace: kserve-ci-e2e-test resourceVersion: "14371" uid: b783d468-e937-4504-b505-f856b5edbce6 kind: Event lastTimestamp: "2026-04-16T18:28:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:33Z" name: error-404-isvc-6138a.18a6e9b37bff221f namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 02d7687e-91ed-47f9-a17a-49d034649faf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6138a namespace: kserve-ci-e2e-test resourceVersion: "14371" uid: b783d468-e937-4504-b505-f856b5edbce6 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-6138a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6138a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-6138a.18a6e9b37c8336bd namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 844c4467-f279-48fb-8960-8d336dfc1e37 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6138a namespace: kserve-ci-e2e-test resourceVersion: "14371" uid: b783d468-e937-4504-b505-f856b5edbce6 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6138a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-6138a.18a6e9b37c83851c namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 11d67af8-163b-4b6a-bd91-0c0e3601bb48 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6138a namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: b783d468-e937-4504-b505-f856b5edbce6 kind: Event lastTimestamp: "2026-04-16T18:28:33Z" message: InferenceService [error-404-isvc-6138a] is Ready metadata: creationTimestamp: "2026-04-16T18:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:33Z" name: error-404-isvc-6138a.18a6e9bfcc0c488b namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: 7a900e70-ee76-44d5-bf44-20dd1241aadf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:38:25.441748Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: 37b339f3-0104-4739-b6fc-8cca0f80fcbd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq.18a6ea49bb65bedc namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: c9459c58-1860-4908-8fd5-a1512dba1b42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: 37b339f3-0104-4739-b6fc-8cca0f80fcbd kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq.18a6ea49c327b113 namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: 41f34ac0-33c9-46ce-9724-10ca7a773cf1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 37b339f3-0104-4739-b6fc-8cca0f80fcbd kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq.18a6ea49c476187d namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: 7c442362-7dc9-4101-ba6c-7851513a89d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 37b339f3-0104-4739-b6fc-8cca0f80fcbd kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq.18a6ea49cd2c47fd namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 4a0ebc2b-c28f-47cb-b544-f38a25619b80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 37b339f3-0104-4739-b6fc-8cca0f80fcbd kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq.18a6ea49cdad07dc namespace: kserve-ci-e2e-test resourceVersion: "19813" uid: 62d626a9-f6d6-4cfd-88ed-6f556416737f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 37b339f3-0104-4739-b6fc-8cca0f80fcbd kind: Event lastTimestamp: "2026-04-16T18:39:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:07Z" name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq.18a6ea49e6397d12 namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 4d7949fe-4a6d-4edf-819f-5e8ba21db8f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 37b339f3-0104-4739-b6fc-8cca0f80fcbd kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq.18a6eacd3f77cf7f namespace: kserve-ci-e2e-test resourceVersion: "24170" uid: 9810fde6-2e5f-4476-9c90-87bea5f3409a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-976d9-predictor-6bd5967d8d namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: de8dc1fe-fce4-4ace-b1ec-61d6b4845681 kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: 'Created pod: error-404-isvc-976d9-predictor-6bd5967d8d-g7jhq' metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-976d9-predictor-6bd5967d8d.18a6ea49baf92119 namespace: kserve-ci-e2e-test resourceVersion: "19785" uid: b9072d64-1b54-4e85-92d3-f831a732df95 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 70eb9f15-57e9-4c06-9356-b925bdfab0f9 kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Scaled up replica set error-404-isvc-976d9-predictor-6bd5967d8d from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-976d9-predictor.18a6ea49b9734bd4 namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 1d7f4ae4-5346-4d66-adad-c6bac7a1b129 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 1c7d399e-cab5-4314-9ddb-3de391870302 kind: Event lastTimestamp: "2026-04-16T18:38:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:55Z" name: error-404-isvc-976d9-predictor.18a6ea4d3843f1d2 namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: 7ef122e8-beec-42b1-b522-7f839eb01a6c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 1c7d399e-cab5-4314-9ddb-3de391870302 kind: Event lastTimestamp: "2026-04-16T18:38:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:55Z" name: error-404-isvc-976d9-predictor.18a6ea4d38ba1ddc namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: e979a38e-7329-4c69-beec-e2d33e5f0001 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:39:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 1c7d399e-cab5-4314-9ddb-3de391870302 kind: Event lastTimestamp: "2026-04-16T18:39:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:40Z" name: error-404-isvc-976d9-predictor.18a6ea543656ba7c namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 28f7ed1d-017d-40a8-8ab1-7be42c7f0b46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:39:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 1c7d399e-cab5-4314-9ddb-3de391870302 kind: Event lastTimestamp: "2026-04-16T18:39:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:40Z" name: error-404-isvc-976d9-predictor.18a6ea5436c99123 namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 76cd2cb4-afd0-4cd5-930c-d02091ade4fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-976d9 namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: ec37eca2-8b4d-4ed9-8b5a-119028f66d1a kind: Event lastTimestamp: "2026-04-16T18:39:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:17Z" name: error-404-isvc-976d9.18a6ea49b96b5d50 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 2449f229-294c-4e1e-84c6-cbefc643aff4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-976d9 namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: ec37eca2-8b4d-4ed9-8b5a-119028f66d1a kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-976d9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-976d9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-976d9.18a6ea49ba11cc98 namespace: kserve-ci-e2e-test resourceVersion: "19781" uid: 1a6aff07-4c12-436a-8023-45f04bb6b1ac reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-976d9 namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: ec37eca2-8b4d-4ed9-8b5a-119028f66d1a kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-976d9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:25Z" name: error-404-isvc-976d9.18a6ea49ba1209b5 namespace: kserve-ci-e2e-test resourceVersion: "19783" uid: 2f69f82a-5dc3-4f10-9e84-1e39e2de29e1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-976d9 namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: ec37eca2-8b4d-4ed9-8b5a-119028f66d1a kind: Event lastTimestamp: "2026-04-16T18:39:17Z" message: InferenceService [error-404-isvc-976d9] is Ready metadata: creationTimestamp: "2026-04-16T18:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:17Z" name: error-404-isvc-976d9.18a6ea55cb718d4a namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: b6da2ee7-8622-4fd4-82ef-e0c8f37e45de reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:48:39.846107Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff namespace: kserve-ci-e2e-test resourceVersion: "24668" uid: e7af7ef2-6558-4525-be3d-8a0e4ae14439 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-979d7-predictor-79bb6798c8-ssnff to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:48:39Z" name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff.18a6ead8c8c04bf3 namespace: kserve-ci-e2e-test resourceVersion: "24672" uid: 0ac4a0f0-ca58-4674-897a-d91563300149 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff namespace: kserve-ci-e2e-test resourceVersion: "24679" uid: e7af7ef2-6558-4525-be3d-8a0e4ae14439 kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:48:39Z" name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff.18a6ead8d08b4690 namespace: kserve-ci-e2e-test resourceVersion: "24687" uid: 655ef34c-93ff-4667-9761-e5c62fd3b065 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: e7af7ef2-6558-4525-be3d-8a0e4ae14439 kind: Event lastTimestamp: "2026-04-16T18:48:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:40Z" name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff.18a6ead8d1ee0e15 namespace: kserve-ci-e2e-test resourceVersion: "24689" uid: 4c12b2ae-b947-4966-bc26-3f38a8e14483 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: e7af7ef2-6558-4525-be3d-8a0e4ae14439 kind: Event lastTimestamp: "2026-04-16T18:48:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:40Z" name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff.18a6ead8da43604c namespace: kserve-ci-e2e-test resourceVersion: "24695" uid: 44268538-9a90-4945-964e-d61913c81841 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: e7af7ef2-6558-4525-be3d-8a0e4ae14439 kind: Event lastTimestamp: "2026-04-16T18:48:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:40Z" name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff.18a6ead8dab7ede5 namespace: kserve-ci-e2e-test resourceVersion: "24697" uid: 03825baa-3506-4c9e-940a-e370fbfbf62f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:48:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: e7af7ef2-6558-4525-be3d-8a0e4ae14439 kind: Event lastTimestamp: "2026-04-16T19:06:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T19:06:11Z" name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff.18a6ead8e3df74ef namespace: kserve-ci-e2e-test resourceVersion: "32743" uid: ab15d1a9-f4a0-4372-989a-acd2171b0b0a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T19:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: e7af7ef2-6558-4525-be3d-8a0e4ae14439 kind: Event lastTimestamp: "2026-04-16T19:06:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T19:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T19:06:09Z" name: error-404-isvc-979d7-predictor-79bb6798c8-ssnff.18a6ebcd2929f8c2 namespace: kserve-ci-e2e-test resourceVersion: "32682" uid: c97567ac-5e9c-4e0f-8301-86d9893d0ab6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-979d7-predictor-79bb6798c8 namespace: kserve-ci-e2e-test resourceVersion: "24661" uid: 83b6e95f-780a-4391-9b89-566b1218418c kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: 'Created pod: error-404-isvc-979d7-predictor-79bb6798c8-ssnff' metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:39Z" name: error-404-isvc-979d7-predictor-79bb6798c8.18a6ead8c8482760 namespace: kserve-ci-e2e-test resourceVersion: "24670" uid: 6a5cea31-d032-4cc5-beb7-1f1605fafed3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24659" uid: 124af8e4-0d02-4f09-beb0-54ac28ae18ee kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: Scaled up replica set error-404-isvc-979d7-predictor-79bb6798c8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:39Z" name: error-404-isvc-979d7-predictor.18a6ead8c6bf16f2 namespace: kserve-ci-e2e-test resourceVersion: "24663" uid: ac9c0f67-addd-432f-bb13-28339d58356c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24660" uid: 75c36032-0921-418b-bab6-10baea652d5d kind: Event lastTimestamp: "2026-04-16T18:49:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:09Z" name: error-404-isvc-979d7-predictor.18a6eadc45a670a1 namespace: kserve-ci-e2e-test resourceVersion: "25010" uid: e17cdaac-bdb0-4fe3-b3e7-1107860914dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24660" uid: 75c36032-0921-418b-bab6-10baea652d5d kind: Event lastTimestamp: "2026-04-16T18:49:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:09Z" name: error-404-isvc-979d7-predictor.18a6eadc461dd05e namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: f3e950a6-0e74-4724-af67-8a5682d00714 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:49:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24856" uid: 75c36032-0921-418b-bab6-10baea652d5d kind: Event lastTimestamp: "2026-04-16T18:49:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:54Z" name: error-404-isvc-979d7-predictor.18a6eae343a4d458 namespace: kserve-ci-e2e-test resourceVersion: "25555" uid: 9b23718e-3017-48d4-bdda-d34d43bf328a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:49:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24856" uid: 75c36032-0921-418b-bab6-10baea652d5d kind: Event lastTimestamp: "2026-04-16T18:49:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:54Z" name: error-404-isvc-979d7-predictor.18a6eae34412dddc namespace: kserve-ci-e2e-test resourceVersion: "25556" uid: 1df17732-2831-4b12-b223-1946c0c34bb6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-979d7 namespace: kserve-ci-e2e-test resourceVersion: "24638" uid: a440672b-a0f3-477a-ad28-d353f4419521 kind: Event lastTimestamp: "2026-04-16T18:49:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:31Z" name: error-404-isvc-979d7.18a6ead8c69b9c31 namespace: kserve-ci-e2e-test resourceVersion: "25334" uid: c0f670f9-b40f-47ad-bcf9-0dfd639ce4a9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-979d7 namespace: kserve-ci-e2e-test resourceVersion: "24638" uid: a440672b-a0f3-477a-ad28-d353f4419521 kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: 'Failed to update status for InferenceService "error-404-isvc-979d7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-979d7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:48:39Z" name: error-404-isvc-979d7.18a6ead8c6ed5200 namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: 9695c301-ee98-48fa-8786-7925348dfd09 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-979d7 namespace: kserve-ci-e2e-test resourceVersion: "24638" uid: a440672b-a0f3-477a-ad28-d353f4419521 kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-979d7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:48:39Z" name: error-404-isvc-979d7.18a6ead8c6ed9671 namespace: kserve-ci-e2e-test resourceVersion: "24666" uid: b3882561-8973-4460-a170-fb8d455cf19d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-979d7 namespace: kserve-ci-e2e-test resourceVersion: "25332" uid: a440672b-a0f3-477a-ad28-d353f4419521 kind: Event lastTimestamp: "2026-04-16T18:49:31Z" message: InferenceService [error-404-isvc-979d7] is Ready metadata: creationTimestamp: "2026-04-16T18:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:31Z" name: error-404-isvc-979d7.18a6eae4c8482382 namespace: kserve-ci-e2e-test resourceVersion: "25333" uid: a1b1ebd4-958a-4062-bef0-38ebbf0c7841 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:29.935961Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: d3bcdd96-015e-40eb-ac17-465d4d2befaa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b16e7-predictor-75968db7b-fhfcp to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:29Z" name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp.18a6e9bf147c8d1e namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: 0d921f6f-5cbb-4429-9fe7-f21f5ff1f630 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: d3bcdd96-015e-40eb-ac17-465d4d2befaa kind: Event lastTimestamp: "2026-04-16T18:28:30Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:30Z" name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp.18a6e9bf1cc0fa40 namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: c1752546-3cf6-4b1f-baae-810c65e04fa9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: d3bcdd96-015e-40eb-ac17-465d4d2befaa kind: Event lastTimestamp: "2026-04-16T18:28:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:30Z" name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp.18a6e9bf1e0561ef namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: b72b9f35-200c-4e90-a42b-34eb6cd23e3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: d3bcdd96-015e-40eb-ac17-465d4d2befaa kind: Event lastTimestamp: "2026-04-16T18:28:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:30Z" name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp.18a6e9bf269a31c6 namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: 71bdcc70-d4c8-49fb-abb2-d1db8428ccd2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: d3bcdd96-015e-40eb-ac17-465d4d2befaa kind: Event lastTimestamp: "2026-04-16T18:28:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:30Z" name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp.18a6e9bf2710d095 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 2d7a05b4-10a5-4654-a553-dc2bbedbbaed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: d3bcdd96-015e-40eb-ac17-465d4d2befaa kind: Event lastTimestamp: "2026-04-16T18:29:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:12Z" name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp.18a6e9bf5b43432a namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: a1b9c179-617b-4fc1-a73d-5dbeef55be4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: d3bcdd96-015e-40eb-ac17-465d4d2befaa kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:54Z" name: error-404-isvc-b16e7-predictor-75968db7b-fhfcp.18a6ea42973487a9 namespace: kserve-ci-e2e-test resourceVersion: "19343" uid: 8fa29096-b35e-4637-be10-d149b283f715 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b16e7-predictor-75968db7b namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: b0593d5e-83b4-4926-89c4-ecb506059517 kind: Event lastTimestamp: "2026-04-16T18:28:29Z" message: 'Created pod: error-404-isvc-b16e7-predictor-75968db7b-fhfcp' metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:29Z" name: error-404-isvc-b16e7-predictor-75968db7b.18a6e9bf140ec6eb namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: 7c33994e-d31f-4d6c-9936-a018f404be5a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 8e1aab82-66e6-4240-8031-c45bc8d6397e kind: Event lastTimestamp: "2026-04-16T18:28:29Z" message: Scaled up replica set error-404-isvc-b16e7-predictor-75968db7b from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:29Z" name: error-404-isvc-b16e7-predictor.18a6e9bf12a1b62f namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: bf01728f-1e76-40e0-ac98-fe0d0d03f031 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 3951f077-c660-4671-84ef-585ad45cbc28 kind: Event lastTimestamp: "2026-04-16T18:28:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:44Z" name: error-404-isvc-b16e7-predictor.18a6e9c291351d5f namespace: kserve-ci-e2e-test resourceVersion: "15196" uid: 6309c067-54f8-46f7-b600-0e03c26f1df8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 3951f077-c660-4671-84ef-585ad45cbc28 kind: Event lastTimestamp: "2026-04-16T18:28:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:44Z" name: error-404-isvc-b16e7-predictor.18a6e9c291abad55 namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: 1aa57666-3d32-45dd-856c-ef8a9cc060f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:28:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 3951f077-c660-4671-84ef-585ad45cbc28 kind: Event lastTimestamp: "2026-04-16T18:29:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:44Z" name: error-404-isvc-b16e7-predictor.18a6e9c61078115f namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 891bb15a-0499-423e-a9f5-f084abd02c76 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:28:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 3951f077-c660-4671-84ef-585ad45cbc28 kind: Event lastTimestamp: "2026-04-16T18:29:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:44Z" name: error-404-isvc-b16e7-predictor.18a6e9c610e9e3dc namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: 44156b9b-4171-4856-b5b6-bf5c489bea6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b16e7 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: d88c6328-e74c-47aa-baf5-b4bea255daaa kind: Event lastTimestamp: "2026-04-16T18:29:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:22Z" name: error-404-isvc-b16e7.18a6e9bf1277ef89 namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 1d9ddfca-c664-4024-83e6-6b33ee3af66e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b16e7 namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: d88c6328-e74c-47aa-baf5-b4bea255daaa kind: Event lastTimestamp: "2026-04-16T18:29:22Z" message: InferenceService [error-404-isvc-b16e7] is Ready metadata: creationTimestamp: "2026-04-16T18:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:22Z" name: error-404-isvc-b16e7.18a6e9cb3ffa605c namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: 82b6e73d-da30-446e-8ed7-75d4d894b45e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:47:50.319440Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ebde8-predictor-97d45866-x4f8w namespace: kserve-ci-e2e-test resourceVersion: "24174" uid: 4e2602a1-85ca-4b15-af41-4de097fe39e7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ebde8-predictor-97d45866-x4f8w to ip-10-0-128-74.ec2.internal metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-ebde8-predictor-97d45866-x4f8w.18a6eacd40bb4b17 namespace: kserve-ci-e2e-test resourceVersion: "24178" uid: 4d9a200d-5389-41b3-9027-d09bdb178eb4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ebde8-predictor-97d45866-x4f8w namespace: kserve-ci-e2e-test resourceVersion: "24182" uid: 4e2602a1-85ca-4b15-af41-4de097fe39e7 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-ebde8-predictor-97d45866-x4f8w.18a6eacd48b029f4 namespace: kserve-ci-e2e-test resourceVersion: "24195" uid: b7775c94-b9e6-4e5a-8c6c-917247aa6737 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ebde8-predictor-97d45866-x4f8w namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: 4e2602a1-85ca-4b15-af41-4de097fe39e7 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-ebde8-predictor-97d45866-x4f8w.18a6eacd49ed0551 namespace: kserve-ci-e2e-test resourceVersion: "24198" uid: 1079e767-e10a-4ccf-93cf-d765ec8c202e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ebde8-predictor-97d45866-x4f8w namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: 4e2602a1-85ca-4b15-af41-4de097fe39e7 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-ebde8-predictor-97d45866-x4f8w.18a6eacd4d4ec2e7 namespace: kserve-ci-e2e-test resourceVersion: "24202" uid: 29e7c060-671f-4a7e-adb9-b255dad87f59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ebde8-predictor-97d45866-x4f8w namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: 4e2602a1-85ca-4b15-af41-4de097fe39e7 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-ebde8-predictor-97d45866-x4f8w.18a6eacd4dbf42e0 namespace: kserve-ci-e2e-test resourceVersion: "24203" uid: d97228aa-e820-41bb-8a05-167e75e268f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ebde8-predictor-97d45866-x4f8w namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: 4e2602a1-85ca-4b15-af41-4de097fe39e7 kind: Event lastTimestamp: "2026-04-16T18:49:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:12Z" name: error-404-isvc-ebde8-predictor-97d45866-x4f8w.18a6eacd706a200b namespace: kserve-ci-e2e-test resourceVersion: "25162" uid: 19ae5b40-6a05-4bbc-8404-3e99b4750fcb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ebde8-predictor-97d45866-x4f8w namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: 4e2602a1-85ca-4b15-af41-4de097fe39e7 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:10Z" name: error-404-isvc-ebde8-predictor-97d45866-x4f8w.18a6eadff36355e8 namespace: kserve-ci-e2e-test resourceVersion: "25104" uid: a0d1f150-60f4-4e91-8dbc-a80964f0d1c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-74.ec2.internal source: component: kubelet host: ip-10-0-128-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ebde8-predictor-97d45866 namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: 4cd5fa2c-908d-4007-a21e-69cc3aa3736d kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: 'Created pod: error-404-isvc-ebde8-predictor-97d45866-x4f8w' metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-ebde8-predictor-97d45866.18a6eacd405e4d87 namespace: kserve-ci-e2e-test resourceVersion: "24177" uid: ca329a84-76b0-4e8e-9764-87c6366cf793 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24156" uid: a9a8c0ab-119e-4fb9-9a6e-223a0d398c58 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Scaled up replica set error-404-isvc-ebde8-predictor-97d45866 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-ebde8-predictor.18a6eacd3eb45c55 namespace: kserve-ci-e2e-test resourceVersion: "24165" uid: 51e7578e-17a4-4bd3-967b-e0ef6213f56d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24161" uid: b970cd11-dd8d-4e59-aa4e-1a48eff04965 kind: Event lastTimestamp: "2026-04-16T18:48:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:05Z" name: error-404-isvc-ebde8-predictor.18a6ead0bd6d5283 namespace: kserve-ci-e2e-test resourceVersion: "24333" uid: d97c0d42-7426-4eda-bb82-24c8caa635e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24161" uid: b970cd11-dd8d-4e59-aa4e-1a48eff04965 kind: Event lastTimestamp: "2026-04-16T18:48:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:05Z" name: error-404-isvc-ebde8-predictor.18a6ead0bddd14f2 namespace: kserve-ci-e2e-test resourceVersion: "24335" uid: 6c0d2ce5-a8a1-4dbc-87d1-08658dda298f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:48:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24334" uid: b970cd11-dd8d-4e59-aa4e-1a48eff04965 kind: Event lastTimestamp: "2026-04-16T18:49:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:05Z" name: error-404-isvc-ebde8-predictor.18a6ead43c949969 namespace: kserve-ci-e2e-test resourceVersion: "24968" uid: d941e09d-cc15-4438-b08d-85d8aba25262 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:48:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24334" uid: b970cd11-dd8d-4e59-aa4e-1a48eff04965 kind: Event lastTimestamp: "2026-04-16T18:49:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:05Z" name: error-404-isvc-ebde8-predictor.18a6ead43d0aa309 namespace: kserve-ci-e2e-test resourceVersion: "24969" uid: c748ca82-5b1e-4ca0-84d7-b192d2f1f850 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: d7812e6e-13f0-4b3f-b3e9-42097023cab2 kind: Event lastTimestamp: "2026-04-16T18:48:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:48:42Z" name: error-404-isvc-ebde8.18a6eacd3ebbad86 namespace: kserve-ci-e2e-test resourceVersion: "24747" uid: cd1c409a-156e-4275-9239-baac70d9c58f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: d7812e6e-13f0-4b3f-b3e9-42097023cab2 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-ebde8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ebde8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-ebde8.18a6eacd3f40a07b namespace: kserve-ci-e2e-test resourceVersion: "24169" uid: 4d654578-8335-4d10-93bc-1a695104b1fb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: d7812e6e-13f0-4b3f-b3e9-42097023cab2 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ebde8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:47:50Z" name: error-404-isvc-ebde8.18a6eacd3f40deb6 namespace: kserve-ci-e2e-test resourceVersion: "24172" uid: f677b13c-1567-48fb-bc9f-f0989f1551ab reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: d7812e6e-13f0-4b3f-b3e9-42097023cab2 kind: Event lastTimestamp: "2026-04-16T18:48:42Z" message: InferenceService [error-404-isvc-ebde8] is Ready metadata: creationTimestamp: "2026-04-16T18:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:48:42Z" name: error-404-isvc-ebde8.18a6ead954f7ebf1 namespace: kserve-ci-e2e-test resourceVersion: "24745" uid: 22c512f3-ffb9-4bf2-a7cc-7ed910f95925 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:26:09.625409Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13294" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-84f646df87-bffbt to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:26:09Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e99e69536300 namespace: kserve-ci-e2e-test resourceVersion: "13300" uid: 2663a7d5-b4cb-4de0-b5fd-aa42ef7b06e3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13306" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:26:10Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e99e8247b868 namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: cb1bf9d6-f46f-4e10-9324-d3bccac7e7fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:7891564b4e51a5ce092a1e9384ed42f60185d9ae7d08ed1c84a6936d5b51591b" metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:10Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e99e839eaf32 namespace: kserve-ci-e2e-test resourceVersion: "13405" uid: 417fb1d4-0b63-46c1-9f98-fbabcff55de3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:26:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:7891564b4e51a5ce092a1e9384ed42f60185d9ae7d08ed1c84a6936d5b51591b" in 4.118s (4.118s including waiting). Image size: 301288371 bytes.' metadata: creationTimestamp: "2026-04-16T18:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:14Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e99f79195cb8 namespace: kserve-ci-e2e-test resourceVersion: "13473" uid: 0bd27e27-1354-42a0-9358-90428b7cd762 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:26:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:14Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e99f7c3ff432 namespace: kserve-ci-e2e-test resourceVersion: "13475" uid: d18fd8e6-e7c5-4d48-b9a2-2b85ef75dacb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:26:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:14Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e99f7c9cc2c1 namespace: kserve-ci-e2e-test resourceVersion: "13478" uid: 01ec90f0-8bfc-4fad-b69e-b6b139bdee25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:26:18Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1324" metadata: creationTimestamp: "2026-04-16T18:26:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:18Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e9a06c7a0611 namespace: kserve-ci-e2e-test resourceVersion: "13548" uid: bc3506a5-16c7-40f3-b426-ff7fc84c56c8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:26:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1324" in 23.322s (23.322s including waiting). Image size: 1560816050 bytes.' metadata: creationTimestamp: "2026-04-16T18:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:41Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e9a5da98b358 namespace: kserve-ci-e2e-test resourceVersion: "13778" uid: 57cefc5c-12b6-4f17-90fe-b3f683c8d2a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:26:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:41Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e9a5dfb7b49d namespace: kserve-ci-e2e-test resourceVersion: "13779" uid: b98e44a4-3476-4d3e-b59b-4436ed09afb1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:26:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:41Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e9a5e14bd895 namespace: kserve-ci-e2e-test resourceVersion: "13780" uid: 0f831b16-d26e-4947-889c-f0b2f8322107 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:28:33Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:33Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e9a608c9e40a namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 5090a5ef-45a7-45e2-954a-bfe220bad84c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: ca1a972f-21c5-48f9-a64f-1cec62e34e77 kind: Event lastTimestamp: "2026-04-16T18:28:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:29Z" name: isvc-sklearn-graph-1-predictor-84f646df87-bffbt.18a6e9bf0fed0c31 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 64dcb5aa-8b76-48cb-b45b-79e90e65383e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-84f646df87 namespace: kserve-ci-e2e-test resourceVersion: "13288" uid: 28e54e85-9da6-4864-869b-7be0b810a4ff kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-84f646df87-bffbt' metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:09Z" name: isvc-sklearn-graph-1-predictor-84f646df87.18a6e99e68cdfc25 namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: cef94584-27ca-4e05-bffa-5a7b710cbc60 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13286" uid: 94bf2331-d8e2-46eb-84c8-a9a8696bb8b8 kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-84f646df87 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:09Z" name: isvc-sklearn-graph-1-predictor.18a6e99e63cfb437 namespace: kserve-ci-e2e-test resourceVersion: "13289" uid: dad547cb-bbf5-4947-9780-f08a7f35cfae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13302" uid: fdfcdfa7-b044-428a-be37-2512f1ea3904 kind: Event lastTimestamp: "2026-04-16T18:27:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:24Z" name: isvc-sklearn-graph-1-predictor.18a6e9a1e9bbb80d namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 09823f70-0892-4524-9ab4-979830d051fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13302" uid: fdfcdfa7-b044-428a-be37-2512f1ea3904 kind: Event lastTimestamp: "2026-04-16T18:27:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:24Z" name: isvc-sklearn-graph-1-predictor.18a6e9a1ea3a008f namespace: kserve-ci-e2e-test resourceVersion: "14143" uid: b64267a7-0b35-4e7a-b3ed-819464f51166 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13624" uid: fdfcdfa7-b044-428a-be37-2512f1ea3904 kind: Event lastTimestamp: "2026-04-16T18:28:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:09Z" name: isvc-sklearn-graph-1-predictor.18a6e9b365abd4bf namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 0982a75e-dcf3-4a81-b8b5-b3b372675880 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13624" uid: fdfcdfa7-b044-428a-be37-2512f1ea3904 kind: Event lastTimestamp: "2026-04-16T18:28:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:09Z" name: isvc-sklearn-graph-1-predictor.18a6e9b36623c889 namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 7431cbc7-2495-4ebf-bb6b-9bbb7b54a46c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: cdff3813-b577-46e7-ab5e-8859d030365e kind: Event lastTimestamp: "2026-04-16T18:27:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:53Z" name: isvc-sklearn-graph-1.18a6e99e6a1694b7 namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: 7e869ad2-c0f8-4f5f-95d5-cb3633c570fc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: cdff3813-b577-46e7-ab5e-8859d030365e kind: Event lastTimestamp: "2026-04-16T18:27:53Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-16T18:27:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:53Z" name: isvc-sklearn-graph-1.18a6e9b695711e30 namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: 98dce25b-2b75-4214-a484-d17caf8da055 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:26:10.207513Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13414" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-555687cc47-vzzzx to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:26:10Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e99e8c059ad9 namespace: kserve-ci-e2e-test resourceVersion: "13418" uid: 4c72b3e5-d073-462c-bba2-e08bc9de8553 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13421" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:26:10Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e99ea8f2c880 namespace: kserve-ci-e2e-test resourceVersion: "13437" uid: 27804c32-2003-4ab0-bc06-f4c807362b3a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:7891564b4e51a5ce092a1e9384ed42f60185d9ae7d08ed1c84a6936d5b51591b" metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:10Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e99eaafceb00 namespace: kserve-ci-e2e-test resourceVersion: "13439" uid: 410d04b4-b8c0-459b-adb1-2420313d3def reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:26:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:7891564b4e51a5ce092a1e9384ed42f60185d9ae7d08ed1c84a6936d5b51591b" in 13.006s (13.006s including waiting). Image size: 301288371 bytes.' metadata: creationTimestamp: "2026-04-16T18:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:23Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e9a1b24046e9 namespace: kserve-ci-e2e-test resourceVersion: "13599" uid: 5928e0fb-f80d-4ab2-a967-2694bd5393a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:26:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:23Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e9a1b50c59e1 namespace: kserve-ci-e2e-test resourceVersion: "13600" uid: 14e1a10f-5861-4250-ba8e-999a86efdc60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:26:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:23Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e9a1b57cd571 namespace: kserve-ci-e2e-test resourceVersion: "13601" uid: c6ffa14e-0cdc-45e3-8fcd-be2624b0523d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:26:27Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1324" metadata: creationTimestamp: "2026-04-16T18:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:27Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e9a283d016b5 namespace: kserve-ci-e2e-test resourceVersion: "13657" uid: 85d7e092-744f-4b20-9c6c-46f2dac7fba3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:26:32Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1324" in 5.554s (5.554s including waiting). Image size: 1560816050 bytes.' metadata: creationTimestamp: "2026-04-16T18:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:32Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e9a3cee9bb94 namespace: kserve-ci-e2e-test resourceVersion: "13697" uid: 3be5ad35-8ae3-497a-a4d6-8c147f2de80f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:26:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:32Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e9a3d36d08fd namespace: kserve-ci-e2e-test resourceVersion: "13698" uid: c98900c6-bdff-461b-b4dc-b179d2a1abe4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:26:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:32Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e9a3d3e68cee namespace: kserve-ci-e2e-test resourceVersion: "13699" uid: c4c39629-e42e-4262-991d-f04abe1c237d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T18:26:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:27:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:34Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e9a3eacc08dd namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: 7e6e17b5-0524-4137-b20f-ff55f52d1e26 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 2bb728d0-b8e7-4e33-8db9-f06483b7bc29 kind: Event lastTimestamp: "2026-04-16T18:28:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:29Z" name: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx.18a6e9bf0c78eaf4 namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: c62e6fc2-a8fd-491b-8d92-03b14eb2ea34 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-555687cc47 namespace: kserve-ci-e2e-test resourceVersion: "13393" uid: e1cce59e-d1a1-47bf-9287-3ba7580e0701 kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-555687cc47-vzzzx' metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:10Z" name: isvc-sklearn-graph-2-predictor-555687cc47.18a6e99e8b8fcf26 namespace: kserve-ci-e2e-test resourceVersion: "13415" uid: e51d9c39-794c-4572-9560-5463f0377c3d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 1112f82e-8340-4104-b18b-c182a601197a kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-555687cc47 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:10Z" name: isvc-sklearn-graph-2-predictor.18a6e99e8016f398 namespace: kserve-ci-e2e-test resourceVersion: "13396" uid: 60a3abce-42b1-4516-801d-bb114024dad0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13392" uid: 68aaf83b-daa1-48ea-bd2b-2e7fc452805b kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: isvc-sklearn-graph-2-predictor.18a6e9a1fef77325 namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: d0c4dd26-fe01-41fe-ae63-0449156992cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13392" uid: 68aaf83b-daa1-48ea-bd2b-2e7fc452805b kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: isvc-sklearn-graph-2-predictor.18a6e9a1ff803221 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: 60d9a0c5-c0d4-403a-9b6f-e3b3e6e9817f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: 68aaf83b-daa1-48ea-bd2b-2e7fc452805b kind: Event lastTimestamp: "2026-04-16T18:27:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:55Z" name: isvc-sklearn-graph-2-predictor.18a6e9ac7c2572b3 namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: c2bc4f7a-ec9f-4538-be21-c47d46e2bf1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: 68aaf83b-daa1-48ea-bd2b-2e7fc452805b kind: Event lastTimestamp: "2026-04-16T18:27:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:55Z" name: isvc-sklearn-graph-2-predictor.18a6e9ac7ca1b4f2 namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: 9c09297a-ed55-45d7-af02-10ed5c0e010f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13374" uid: 62529c0f-b0f6-4a5f-94d5-f3ace6604d1a kind: Event lastTimestamp: "2026-04-16T18:27:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:44Z" name: isvc-sklearn-graph-2.18a6e99e7ff2285a namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: beef5cde-f5e1-4ac5-839a-5b403d390d0b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 62529c0f-b0f6-4a5f-94d5-f3ace6604d1a kind: Event lastTimestamp: "2026-04-16T18:27:44Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-16T18:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:44Z" name: isvc-sklearn-graph-2.18a6e9b477b2fbc6 namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 8532ba53-4d00-4b69-8f47-fabfe5017885 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:26:09.866781Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13350" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-kl7jj to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:26:09Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e99e77b67693 namespace: kserve-ci-e2e-test resourceVersion: "13355" uid: fe30581b-6ae7-4079-9ca4-5ecd2f5d0664 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13365" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:26:10Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e99e905b3177 namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: 8c1b4e5b-2dfd-4e2c-883f-b5d881ea5889 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:26:10Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:7891564b4e51a5ce092a1e9384ed42f60185d9ae7d08ed1c84a6936d5b51591b" metadata: creationTimestamp: "2026-04-16T18:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:10Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e99e91c0f235 namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: f377e87a-f74e-40ae-81af-831cb7a48c7b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:26:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:7891564b4e51a5ce092a1e9384ed42f60185d9ae7d08ed1c84a6936d5b51591b" in 3.872s (3.872s including waiting). Image size: 301288371 bytes.' metadata: creationTimestamp: "2026-04-16T18:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:14Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e99f78946c38 namespace: kserve-ci-e2e-test resourceVersion: "13472" uid: 0423afdb-f1b0-4b4e-aea7-5110c35a163a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:26:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:14Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e99f7c21fbc2 namespace: kserve-ci-e2e-test resourceVersion: "13474" uid: a305be57-64c1-4202-987d-ffc13b3a786b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:26:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:14Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e99f7c865fee namespace: kserve-ci-e2e-test resourceVersion: "13476" uid: 5f53eda2-e5c4-493f-96e2-1e3691af8447 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:26:18Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T18:26:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:18Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e9a06c9269c6 namespace: kserve-ci-e2e-test resourceVersion: "13550" uid: 01ae6c74-3996-4de0-89fd-77558c6a62dc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:26:41Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.701s (23.701s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T18:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:41Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e9a5f145aa69 namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: 513642b9-d185-46d0-987a-fcc6aa1dfb66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:26:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:42Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e9a5f5645b97 namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: 2fe417cf-663d-4ae9-ade0-2009170eaf34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:26:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:42Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e9a5f662d8f9 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 18ba64d5-778c-44da-9ab4-5b37aa154f14 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:27:33Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:33Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e9a608b86cb3 namespace: kserve-ci-e2e-test resourceVersion: "14254" uid: f31afaa7-2f5c-47f8-a09a-50f34f9b661b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: e3c6afc8-1f23-4511-9ae7-0a13e296333e kind: Event lastTimestamp: "2026-04-16T18:28:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:29Z" name: isvc-xgboost-graph-predictor-784cb989b8-kl7jj.18a6e9bf15d8c7de namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: 94c2e805-622c-4d5f-a6f2-2ddddd09c638 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "13338" uid: f20bf89f-768d-42f8-bc23-7febe1c80e36 kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-kl7jj' metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:09Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a6e99e774146dc namespace: kserve-ci-e2e-test resourceVersion: "13352" uid: 9318f692-7f6d-46d6-a9cb-b2466b7c40d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13332" uid: 7e408d9b-4c14-4969-9565-ffb7e2628ddc kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:09Z" name: isvc-xgboost-graph-predictor.18a6e99e7479b8aa namespace: kserve-ci-e2e-test resourceVersion: "13340" uid: 56a38221-36e3-4e6c-b811-8a61d34399ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13336" uid: 46d5fa3b-f01e-4e7f-85a6-4d847719a49b kind: Event lastTimestamp: "2026-04-16T18:26:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:54Z" name: isvc-xgboost-graph-predictor.18a6e9a1f3539656 namespace: kserve-ci-e2e-test resourceVersion: "13891" uid: 6199057d-93ef-41f9-aa86-1f962ca05c68 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13336" uid: 46d5fa3b-f01e-4e7f-85a6-4d847719a49b kind: Event lastTimestamp: "2026-04-16T18:26:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:54Z" name: isvc-xgboost-graph-predictor.18a6e9a1f3e2d0dc namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: cd76d726-3755-4c80-ba6f-11378c8349fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:27:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13630" uid: 46d5fa3b-f01e-4e7f-85a6-4d847719a49b kind: Event lastTimestamp: "2026-04-16T18:28:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:09Z" name: isvc-xgboost-graph-predictor.18a6e9ac70574681 namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: e36243b6-026a-4850-8ef5-d1442d64898e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:27:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13630" uid: 46d5fa3b-f01e-4e7f-85a6-4d847719a49b kind: Event lastTimestamp: "2026-04-16T18:28:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:09Z" name: isvc-xgboost-graph-predictor.18a6e9ac70c3a053 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: 8afb1dca-6303-4c46-9ecc-37f9d118e6b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13325" uid: 46fc0564-6e10-40d0-8d06-5f88b09e9914 kind: Event lastTimestamp: "2026-04-16T18:27:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:43Z" name: isvc-xgboost-graph.18a6e99e747b3bec namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: 33b877a9-1b2c-4626-b155-eac1f7b25f55 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 46fc0564-6e10-40d0-8d06-5f88b09e9914 kind: Event lastTimestamp: "2026-04-16T18:27:43Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-16T18:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:43Z" name: isvc-xgboost-graph.18a6e9b44122885b namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: a8406b5c-3b78-4d0b-b069-0cda99868189 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:19.628971Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-79c585885d-665ck namespace: kserve-ci-e2e-test resourceVersion: "14797" uid: 39df0897-8c7c-479b-8316-1650efd08fe8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-79c585885d-665ck to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:19Z" name: model-chainer-79c585885d-665ck.18a6e9bcae247da8 namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: 822bd619-bf98-4098-80a6-0b24b5314878 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-79c585885d-665ck namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: 39df0897-8c7c-479b-8316-1650efd08fe8 kind: Event lastTimestamp: "2026-04-16T18:28:20Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:20Z" name: model-chainer-79c585885d-665ck.18a6e9bcc71d052b namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: fff4d9c4-cd05-426c-9118-41b18ec55aa5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79c585885d-665ck namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 39df0897-8c7c-479b-8316-1650efd08fe8 kind: Event lastTimestamp: "2026-04-16T18:28:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:20Z" name: model-chainer-79c585885d-665ck.18a6e9bcc85d2240 namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: d29eb4cc-bd06-418a-b054-14722914d7ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79c585885d-665ck namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 39df0897-8c7c-479b-8316-1650efd08fe8 kind: Event lastTimestamp: "2026-04-16T18:28:20Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-16T18:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:20Z" name: model-chainer-79c585885d-665ck.18a6e9bccaf52c81 namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: 1db5e5b1-894a-4e35-8af3-4d0680ea3d05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79c585885d-665ck namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 39df0897-8c7c-479b-8316-1650efd08fe8 kind: Event lastTimestamp: "2026-04-16T18:28:20Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-16T18:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:20Z" name: model-chainer-79c585885d-665ck.18a6e9bccb61abb9 namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 7838802e-854d-47ec-a6e5-1409956e37f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79c585885d-665ck namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 39df0897-8c7c-479b-8316-1650efd08fe8 kind: Event lastTimestamp: "2026-04-16T18:28:29Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:29Z" name: model-chainer-79c585885d-665ck.18a6e9bf06821dd5 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: f397068d-6b07-4118-9c13-a2e0486bcd70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-79c585885d-665ck namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 39df0897-8c7c-479b-8316-1650efd08fe8 kind: Event lastTimestamp: "2026-04-16T18:28:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:56Z" name: model-chainer-79c585885d-665ck.18a6e9bf7ac7c5ec namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 0b9838e8-5c5c-40be-acbf-1a8a616b263a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-79c585885d namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: 4d313f6b-17de-43db-a587-552bc0e38c1e kind: Event lastTimestamp: "2026-04-16T18:28:19Z" message: 'Created pod: model-chainer-79c585885d-665ck' metadata: creationTimestamp: "2026-04-16T18:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:19Z" name: model-chainer-79c585885d.18a6e9bcad9240b0 namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: 3ba66205-fc0c-4a80-9f7a-fdfa6adbe76d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 09d6c2e1-df61-4fbb-96ea-a74d453c395c kind: Event lastTimestamp: "2026-04-16T18:28:19Z" message: Scaled up replica set model-chainer-79c585885d from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:19Z" name: model-chainer.18a6e9bcacb7fbd3 namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 82db3846-6b5f-498f-a78f-a361222e0ad4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 23eadad3-5870-4528-a86d-168ed0c172cc kind: Event lastTimestamp: "2026-04-16T18:28:19Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-16T18:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:19Z" name: model-chainer.18a6e9bcb2d7c628 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 9ad26961-8321-4a01-915e-371a4e2acb15 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 23eadad3-5870-4528-a86d-168ed0c172cc kind: Event lastTimestamp: "2026-04-16T18:28:19Z" message: 'Failed to update status for InferenceGraph "model-chainer": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:19Z" name: model-chainer.18a6e9bcb572c3a4 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 815595a5-eb38-4db8-92cd-696c8fdaa68d reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 23eadad3-5870-4528-a86d-168ed0c172cc kind: Event lastTimestamp: "2026-04-16T18:28:19Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:19Z" name: model-chainer.18a6e9bcb5733355 namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 0bf8a1a4-f204-46cd-9415-a429622a31f8 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:39:04.739158Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0f5d9-59b945f668-6xvgw namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: d54043a1-da58-445d-9389-11833807c48e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0f5d9-59b945f668-6xvgw to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:39:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:39:04Z" name: sequence-graph-0f5d9-59b945f668-6xvgw.18a6ea52e1b4a678 namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: f360199b-1524-444e-aa51-771538aa66e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:04Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0f5d9-59b945f668-6xvgw namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: d54043a1-da58-445d-9389-11833807c48e kind: Event lastTimestamp: "2026-04-16T18:39:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-0f5d9-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:04Z" name: sequence-graph-0f5d9-59b945f668-6xvgw.18a6ea52f0df5cb8 namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: dcde2771-ff82-4e7e-8908-f150e5a6a12e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:05Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0f5d9-59b945f668-6xvgw namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: d54043a1-da58-445d-9389-11833807c48e kind: Event lastTimestamp: "2026-04-16T18:39:05Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:39:05Z" name: sequence-graph-0f5d9-59b945f668-6xvgw.18a6ea531e6415c9 namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 9114bf9c-0262-4aea-9cb7-a2f7bee08d94 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d9} kind: Pod name: sequence-graph-0f5d9-59b945f668-6xvgw namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: d54043a1-da58-445d-9389-11833807c48e kind: Event lastTimestamp: "2026-04-16T18:39:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:05Z" name: sequence-graph-0f5d9-59b945f668-6xvgw.18a6ea531fbbf7d4 namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: c060d127-d943-4157-a2de-f3c7c93aa7ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d9} kind: Pod name: sequence-graph-0f5d9-59b945f668-6xvgw namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: d54043a1-da58-445d-9389-11833807c48e kind: Event lastTimestamp: "2026-04-16T18:39:05Z" message: 'Created container: sequence-graph-0f5d9' metadata: creationTimestamp: "2026-04-16T18:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:05Z" name: sequence-graph-0f5d9-59b945f668-6xvgw.18a6ea53227008ef namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 71c1b574-1d7e-4c86-90a1-26673c6fbaf2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d9} kind: Pod name: sequence-graph-0f5d9-59b945f668-6xvgw namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: d54043a1-da58-445d-9389-11833807c48e kind: Event lastTimestamp: "2026-04-16T18:39:05Z" message: Started container sequence-graph-0f5d9 metadata: creationTimestamp: "2026-04-16T18:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:05Z" name: sequence-graph-0f5d9-59b945f668-6xvgw.18a6ea5322dba37c namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 1cfe0203-c357-47dd-bd32-da09efa3fff1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d9} kind: Pod name: sequence-graph-0f5d9-59b945f668-6xvgw namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: d54043a1-da58-445d-9389-11833807c48e kind: Event lastTimestamp: "2026-04-16T18:39:14Z" message: Stopping container sequence-graph-0f5d9 metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:14Z" name: sequence-graph-0f5d9-59b945f668-6xvgw.18a6ea5539cd3e1c namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: b227cc12-124d-4067-a84f-d0e2ca1e0db1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d9} kind: Pod name: sequence-graph-0f5d9-59b945f668-6xvgw namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: d54043a1-da58-445d-9389-11833807c48e kind: Event lastTimestamp: "2026-04-16T18:39:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:42Z" name: sequence-graph-0f5d9-59b945f668-6xvgw.18a6ea55d7fd7627 namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: a66ee9c7-5be7-487b-9055-272669a00703 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0f5d9-59b945f668 namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: b60b7b8f-e4da-4e68-be86-1a80f27bbf60 kind: Event lastTimestamp: "2026-04-16T18:39:04Z" message: 'Created pod: sequence-graph-0f5d9-59b945f668-6xvgw' metadata: creationTimestamp: "2026-04-16T18:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:04Z" name: sequence-graph-0f5d9-59b945f668.18a6ea52e145331c namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: a85dd63d-b60a-4be9-90f2-5ab740e144ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 7d458227-5e4f-498e-b9a5-cf1a8355f995 kind: Event lastTimestamp: "2026-04-16T18:39:04Z" message: Scaled up replica set sequence-graph-0f5d9-59b945f668 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:04Z" name: sequence-graph-0f5d9.18a6ea52e06fa53a namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: 152f2206-a5cb-4b35-a746-a26fc3fafb4a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: fc703ebf-fd9a-49e3-8518-74733edcafe5 kind: Event lastTimestamp: "2026-04-16T18:39:04Z" message: InferenceGraph [sequence-graph-0f5d9] is Ready metadata: creationTimestamp: "2026-04-16T18:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:04Z" name: sequence-graph-0f5d9.18a6ea52e656801e namespace: kserve-ci-e2e-test resourceVersion: "20178" uid: 0031bee8-a10a-4b8b-a14e-4c102e2dd83d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "20152" uid: fc703ebf-fd9a-49e3-8518-74733edcafe5 kind: Event lastTimestamp: "2026-04-16T18:39:04Z" message: 'Failed to update status for InferenceGraph "sequence-graph-0f5d9": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-0f5d9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:04Z" name: sequence-graph-0f5d9.18a6ea52e940af62 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: 1449401d-4d6c-4bdd-8342-1508d9385c8a reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "20152" uid: fc703ebf-fd9a-49e3-8518-74733edcafe5 kind: Event lastTimestamp: "2026-04-16T18:39:04Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-0f5d9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:04Z" name: sequence-graph-0f5d9.18a6ea52e940f0e1 namespace: kserve-ci-e2e-test resourceVersion: "20180" uid: c2c7397e-ae9c-43f4-b518-80f12dc9ba59 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:40:25.020380Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5aeb1-658b497c45-rh6fs namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 5d32adf8-83ce-4e65-b767-0421700df8b6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-5aeb1-658b497c45-rh6fs to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:40:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:40:25Z" name: sequence-graph-5aeb1-658b497c45-rh6fs.18a6ea6592d6d40b namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: 2067a984-1287-4562-ae11-1f01deac1d24 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:25Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5aeb1-658b497c45-rh6fs namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 5d32adf8-83ce-4e65-b767-0421700df8b6 kind: Event lastTimestamp: "2026-04-16T18:40:25Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:40:25Z" name: sequence-graph-5aeb1-658b497c45-rh6fs.18a6ea65ab7926e9 namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: b0a6d3fc-87da-45f4-8fd6-e931162fa582 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5aeb1} kind: Pod name: sequence-graph-5aeb1-658b497c45-rh6fs namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 5d32adf8-83ce-4e65-b767-0421700df8b6 kind: Event lastTimestamp: "2026-04-16T18:40:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:25Z" name: sequence-graph-5aeb1-658b497c45-rh6fs.18a6ea65ad08e6bd namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: e3f5e203-1eb9-4987-bd49-fea8ae381c86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5aeb1} kind: Pod name: sequence-graph-5aeb1-658b497c45-rh6fs namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 5d32adf8-83ce-4e65-b767-0421700df8b6 kind: Event lastTimestamp: "2026-04-16T18:40:25Z" message: 'Created container: sequence-graph-5aeb1' metadata: creationTimestamp: "2026-04-16T18:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:25Z" name: sequence-graph-5aeb1-658b497c45-rh6fs.18a6ea65af8c5d94 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 2cd9aa06-0a75-456b-b426-8844ab060ddb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5aeb1} kind: Pod name: sequence-graph-5aeb1-658b497c45-rh6fs namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 5d32adf8-83ce-4e65-b767-0421700df8b6 kind: Event lastTimestamp: "2026-04-16T18:40:25Z" message: Started container sequence-graph-5aeb1 metadata: creationTimestamp: "2026-04-16T18:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:40:25Z" name: sequence-graph-5aeb1-658b497c45-rh6fs.18a6ea65aff7d4d2 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: edb67aad-1efc-42b2-86b8-0af5cd8bc3e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5aeb1} kind: Pod name: sequence-graph-5aeb1-658b497c45-rh6fs namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 5d32adf8-83ce-4e65-b767-0421700df8b6 kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: Stopping container sequence-graph-5aeb1 metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:39Z" name: sequence-graph-5aeb1-658b497c45-rh6fs.18a6ead8bc5c9688 namespace: kserve-ci-e2e-test resourceVersion: "24600" uid: 41ff028e-b0c7-4305-8502-dfd1e24ef63e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5aeb1} kind: Pod name: sequence-graph-5aeb1-658b497c45-rh6fs namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 5d32adf8-83ce-4e65-b767-0421700df8b6 kind: Event lastTimestamp: "2026-04-16T18:49:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:06Z" name: sequence-graph-5aeb1-658b497c45-rh6fs.18a6ead93678bc11 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: 8d000be1-f5fc-43d0-b87f-08650b839160 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-5aeb1-658b497c45 namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: f4614bd8-4a7e-4503-a17e-700acea85594 kind: Event lastTimestamp: "2026-04-16T18:40:25Z" message: 'Created pod: sequence-graph-5aeb1-658b497c45-rh6fs' metadata: creationTimestamp: "2026-04-16T18:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:25Z" name: sequence-graph-5aeb1-658b497c45.18a6ea65923fb7a6 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 7579e21b-a8b2-4d77-9f2f-c3d5c721f16a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: 2780e494-48a6-4013-a546-869e519e1317 kind: Event lastTimestamp: "2026-04-16T18:40:24Z" message: Scaled up replica set sequence-graph-5aeb1-658b497c45 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:24Z" name: sequence-graph-5aeb1.18a6ea659177fb92 namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: 0f288862-5dbb-4ed6-93dd-73ce825f8b67 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 99fecaef-f54c-499b-a0f9-ea16d27684aa kind: Event lastTimestamp: "2026-04-16T18:40:25Z" message: InferenceGraph [sequence-graph-5aeb1] is Ready metadata: creationTimestamp: "2026-04-16T18:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:25Z" name: sequence-graph-5aeb1.18a6ea659737b265 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: b3b23957-2242-4129-a472-4cbaebb96de8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:40:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 4deeee7c-0a85-4c46-a178-142174c977d8 kind: Event lastTimestamp: "2026-04-16T18:40:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:55Z" name: sequence-graph-5aeb1.18a6ea6910046c3f namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: eb522cd4-1abe-491e-ba57-670e29aca255 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:40:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 4deeee7c-0a85-4c46-a178-142174c977d8 kind: Event lastTimestamp: "2026-04-16T18:40:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:55Z" name: sequence-graph-5aeb1.18a6ea69109b514b namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 48e8d52e-a769-4dfa-aa0a-380b7fc59d45 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:29:39.905529Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b16e7-649b6f987b-vl7zs namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 4faf4db5-9379-4b3d-aa7f-b94616705992 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-b16e7-649b6f987b-vl7zs to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:29:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:29:39Z" name: sequence-graph-b16e7-649b6f987b-vl7zs.18a6e9cf5eff6b0a namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: c2e3134d-f196-486b-9df7-5e5302d0b684 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:40Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b16e7-649b6f987b-vl7zs namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 4faf4db5-9379-4b3d-aa7f-b94616705992 kind: Event lastTimestamp: "2026-04-16T18:29:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-b16e7-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:40Z" name: sequence-graph-b16e7-649b6f987b-vl7zs.18a6e9cf6930b2df namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: e0819157-01f6-4b9c-bcbf-b8c9bc8618f4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:40Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b16e7-649b6f987b-vl7zs namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: 4faf4db5-9379-4b3d-aa7f-b94616705992 kind: Event lastTimestamp: "2026-04-16T18:29:40Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:29:40Z" name: sequence-graph-b16e7-649b6f987b-vl7zs.18a6e9cf9b9b0786 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 017dce8d-6f84-453c-9bdc-0d7ba27ee4da reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b16e7} kind: Pod name: sequence-graph-b16e7-649b6f987b-vl7zs namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 4faf4db5-9379-4b3d-aa7f-b94616705992 kind: Event lastTimestamp: "2026-04-16T18:29:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:40Z" name: sequence-graph-b16e7-649b6f987b-vl7zs.18a6e9cf9cf70383 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 9e088349-adc5-484a-b2c5-3026ce4458ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b16e7} kind: Pod name: sequence-graph-b16e7-649b6f987b-vl7zs namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 4faf4db5-9379-4b3d-aa7f-b94616705992 kind: Event lastTimestamp: "2026-04-16T18:29:40Z" message: 'Created container: sequence-graph-b16e7' metadata: creationTimestamp: "2026-04-16T18:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:40Z" name: sequence-graph-b16e7-649b6f987b-vl7zs.18a6e9cf9ff2a0c2 namespace: kserve-ci-e2e-test resourceVersion: "15716" uid: 5d951aa2-5d0f-49fa-bdf9-4207a433cd5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b16e7} kind: Pod name: sequence-graph-b16e7-649b6f987b-vl7zs namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 4faf4db5-9379-4b3d-aa7f-b94616705992 kind: Event lastTimestamp: "2026-04-16T18:29:41Z" message: Started container sequence-graph-b16e7 metadata: creationTimestamp: "2026-04-16T18:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:41Z" name: sequence-graph-b16e7-649b6f987b-vl7zs.18a6e9cfa05ee636 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 956f5ccf-b83b-488e-9456-126b55e0e89a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b16e7} kind: Pod name: sequence-graph-b16e7-649b6f987b-vl7zs namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 4faf4db5-9379-4b3d-aa7f-b94616705992 kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Stopping container sequence-graph-b16e7 metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:54Z" name: sequence-graph-b16e7-649b6f987b-vl7zs.18a6ea428ad5e1fe namespace: kserve-ci-e2e-test resourceVersion: "19273" uid: 3710632e-bfb1-4446-8d82-efc172032e05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b16e7} kind: Pod name: sequence-graph-b16e7-649b6f987b-vl7zs namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 4faf4db5-9379-4b3d-aa7f-b94616705992 kind: Event lastTimestamp: "2026-04-16T18:38:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:22Z" name: sequence-graph-b16e7-649b6f987b-vl7zs.18a6ea43518d1a96 namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 18ba4bb2-8ee5-4900-9fa0-a46efa8c0945 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-b16e7-649b6f987b namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 6cc10253-17e3-4be7-aaf5-9f9f5a80ecda kind: Event lastTimestamp: "2026-04-16T18:29:39Z" message: 'Created pod: sequence-graph-b16e7-649b6f987b-vl7zs' metadata: creationTimestamp: "2026-04-16T18:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:39Z" name: sequence-graph-b16e7-649b6f987b.18a6e9cf5e7b2bca namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 2d3d1a57-ffde-4c20-84dd-2e17618dbac3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-b16e7 namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: 7f2b9576-4b3c-415a-a8a1-98c3da2bd521 kind: Event lastTimestamp: "2026-04-16T18:29:39Z" message: Scaled up replica set sequence-graph-b16e7-649b6f987b from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:39Z" name: sequence-graph-b16e7.18a6e9cf5d80cb3a namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: dcfc4e71-ed05-4e06-81c4-75bb68ae237d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b16e7 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: 0492c1e9-d45c-4f18-8aa9-3f4e99280255 kind: Event lastTimestamp: "2026-04-16T18:29:39Z" message: InferenceGraph [sequence-graph-b16e7] is Ready metadata: creationTimestamp: "2026-04-16T18:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:39Z" name: sequence-graph-b16e7.18a6e9cf61e5c295 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: c4b5970e-8e6f-48c0-8535-20286f52399e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:29:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b16e7 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: e7d7d70e-f527-4ec9-a7bf-4cafcddadeec kind: Event lastTimestamp: "2026-04-16T18:30:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:24Z" name: sequence-graph-b16e7.18a6e9d2dc8c9392 namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 7c4b3624-20d7-4b1f-af05-e1659b3be49f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:29:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b16e7 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: e7d7d70e-f527-4ec9-a7bf-4cafcddadeec kind: Event lastTimestamp: "2026-04-16T18:30:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:24Z" name: sequence-graph-b16e7.18a6e9d2dcfa71ba namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 0896d92d-2f16-4390-8182-004a774e37ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:50:20.623412Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-56ec1-696b76c44b-v9dvq namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: d67a998f-616d-44a6-b8df-ff46590a864e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-56ec1-696b76c44b-v9dvq to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:50:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:50:20Z" name: splitter-graph-56ec1-696b76c44b-v9dvq.18a6eaf03f8bde6b namespace: kserve-ci-e2e-test resourceVersion: "25945" uid: e3c13bdf-46ca-472e-9bd0-66040cf62ad8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:20Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-56ec1-696b76c44b-v9dvq namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: d67a998f-616d-44a6-b8df-ff46590a864e kind: Event lastTimestamp: "2026-04-16T18:50:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-56ec1-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:50:20Z" name: splitter-graph-56ec1-696b76c44b-v9dvq.18a6eaf04b75fbee namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 3282c88d-8a3f-4b3f-9a9d-1ed1b53f31eb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:21Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-56ec1-696b76c44b-v9dvq namespace: kserve-ci-e2e-test resourceVersion: "25948" uid: d67a998f-616d-44a6-b8df-ff46590a864e kind: Event lastTimestamp: "2026-04-16T18:50:21Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:50:21Z" name: splitter-graph-56ec1-696b76c44b-v9dvq.18a6eaf07c690aca namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: a570a35b-edec-467b-92b1-5151b16263b1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56ec1} kind: Pod name: splitter-graph-56ec1-696b76c44b-v9dvq namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: d67a998f-616d-44a6-b8df-ff46590a864e kind: Event lastTimestamp: "2026-04-16T18:50:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:50:21Z" name: splitter-graph-56ec1-696b76c44b-v9dvq.18a6eaf07dae15be namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: 3dbf20f4-5e38-4037-beb3-cf433d095af0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56ec1} kind: Pod name: splitter-graph-56ec1-696b76c44b-v9dvq namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: d67a998f-616d-44a6-b8df-ff46590a864e kind: Event lastTimestamp: "2026-04-16T18:50:21Z" message: 'Created container: splitter-graph-56ec1' metadata: creationTimestamp: "2026-04-16T18:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:50:21Z" name: splitter-graph-56ec1-696b76c44b-v9dvq.18a6eaf0808420bd namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 4edb4061-63be-4d1b-bd68-1ee1b542f3bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56ec1} kind: Pod name: splitter-graph-56ec1-696b76c44b-v9dvq namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: d67a998f-616d-44a6-b8df-ff46590a864e kind: Event lastTimestamp: "2026-04-16T18:50:21Z" message: Started container splitter-graph-56ec1 metadata: creationTimestamp: "2026-04-16T18:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:50:21Z" name: splitter-graph-56ec1-696b76c44b-v9dvq.18a6eaf080edea54 namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: 4def4f88-ed03-4735-aaaa-5cff4743bac2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:58:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56ec1} kind: Pod name: splitter-graph-56ec1-696b76c44b-v9dvq namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: d67a998f-616d-44a6-b8df-ff46590a864e kind: Event lastTimestamp: "2026-04-16T18:58:35Z" message: Stopping container splitter-graph-56ec1 metadata: creationTimestamp: "2026-04-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:58:35Z" name: splitter-graph-56ec1-696b76c44b-v9dvq.18a6eb636bfe1222 namespace: kserve-ci-e2e-test resourceVersion: "29532" uid: 66c7a165-6e4e-4a02-91cc-01250de85fac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56ec1} kind: Pod name: splitter-graph-56ec1-696b76c44b-v9dvq namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: d67a998f-616d-44a6-b8df-ff46590a864e kind: Event lastTimestamp: "2026-04-16T18:59:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:59:03Z" name: splitter-graph-56ec1-696b76c44b-v9dvq.18a6eb6422ca3959 namespace: kserve-ci-e2e-test resourceVersion: "29763" uid: 9fb04f71-2b0a-4128-8e3a-b58f55dddfff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-56ec1-696b76c44b namespace: kserve-ci-e2e-test resourceVersion: "25937" uid: 5ddf037f-a894-4d54-9893-3f0c42dea1e9 kind: Event lastTimestamp: "2026-04-16T18:50:20Z" message: 'Created pod: splitter-graph-56ec1-696b76c44b-v9dvq' metadata: creationTimestamp: "2026-04-16T18:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:20Z" name: splitter-graph-56ec1-696b76c44b.18a6eaf03ef18c3b namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 16e4e10c-c1fe-416d-b660-a8a48c0ddaa4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: a276beb2-1cf0-466a-bddf-1dff343820d9 kind: Event lastTimestamp: "2026-04-16T18:50:20Z" message: Scaled up replica set splitter-graph-56ec1-696b76c44b from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:20Z" name: splitter-graph-56ec1.18a6eaf03e0e7213 namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: 8fc40f60-5998-4fb7-83c6-fd312ca048d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: e87511d2-ef8e-4356-9cdf-048a8eb8d50a kind: Event lastTimestamp: "2026-04-16T18:50:20Z" message: InferenceGraph [splitter-graph-56ec1] is Ready metadata: creationTimestamp: "2026-04-16T18:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:50:20Z" name: splitter-graph-56ec1.18a6eaf043979900 namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: 45aa4881-6bb5-4b0a-aef6-feb0edcab6ec reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: e87511d2-ef8e-4356-9cdf-048a8eb8d50a kind: Event lastTimestamp: "2026-04-16T18:50:20Z" message: 'Failed to update status for InferenceGraph "splitter-graph-56ec1": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-56ec1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:50:20Z" name: splitter-graph-56ec1.18a6eaf045fcb4db namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: 84200faa-f48d-45d7-9806-960bfa51736f reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: e87511d2-ef8e-4356-9cdf-048a8eb8d50a kind: Event lastTimestamp: "2026-04-16T18:50:20Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-56ec1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:50:20Z" name: splitter-graph-56ec1.18a6eaf045fceec3 namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: f4ee87c7-581b-44a6-8c8d-835769caea8c reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:50:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 2c5362af-20a4-4543-a675-323a473bfc2c kind: Event lastTimestamp: "2026-04-16T18:50:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:50:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:50Z" name: splitter-graph-56ec1.18a6eaf3bc818642 namespace: kserve-ci-e2e-test resourceVersion: "26324" uid: 5deb7980-7856-4afd-9f97-3b2cc73a3868 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:50:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 2c5362af-20a4-4543-a675-323a473bfc2c kind: Event lastTimestamp: "2026-04-16T18:50:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:50:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:50Z" name: splitter-graph-56ec1.18a6eaf3bcf48d2f namespace: kserve-ci-e2e-test resourceVersion: "26325" uid: fb3021f3-dd1e-496d-8064-de51cb2947cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:49:00.336124Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ebde8-7c98c9c67-6zzq2 namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: 48fd4505-0055-468b-911d-1f6feecd7faa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-ebde8-7c98c9c67-6zzq2 to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:49:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:49:00Z" name: splitter-graph-ebde8-7c98c9c67-6zzq2.18a6eadd8e0d12bd namespace: kserve-ci-e2e-test resourceVersion: "24910" uid: 47ed7d50-6df8-4a89-97e4-c87687a9a637 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:00Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ebde8-7c98c9c67-6zzq2 namespace: kserve-ci-e2e-test resourceVersion: "24914" uid: 48fd4505-0055-468b-911d-1f6feecd7faa kind: Event lastTimestamp: "2026-04-16T18:49:00Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:49:00Z" name: splitter-graph-ebde8-7c98c9c67-6zzq2.18a6eadda6be4fa0 namespace: kserve-ci-e2e-test resourceVersion: "24924" uid: 533dee2a-9f73-4814-bcf7-e72065d581f7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ebde8} kind: Pod name: splitter-graph-ebde8-7c98c9c67-6zzq2 namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 48fd4505-0055-468b-911d-1f6feecd7faa kind: Event lastTimestamp: "2026-04-16T18:49:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:00Z" name: splitter-graph-ebde8-7c98c9c67-6zzq2.18a6eadda80edca9 namespace: kserve-ci-e2e-test resourceVersion: "24926" uid: 5a755aaa-e932-46b1-9583-b7538524c7ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ebde8} kind: Pod name: splitter-graph-ebde8-7c98c9c67-6zzq2 namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 48fd4505-0055-468b-911d-1f6feecd7faa kind: Event lastTimestamp: "2026-04-16T18:49:00Z" message: 'Created container: splitter-graph-ebde8' metadata: creationTimestamp: "2026-04-16T18:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:00Z" name: splitter-graph-ebde8-7c98c9c67-6zzq2.18a6eaddaaa76a2e namespace: kserve-ci-e2e-test resourceVersion: "24928" uid: 70141e8a-f282-4d71-a9d6-fc18f497a711 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ebde8} kind: Pod name: splitter-graph-ebde8-7c98c9c67-6zzq2 namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 48fd4505-0055-468b-911d-1f6feecd7faa kind: Event lastTimestamp: "2026-04-16T18:49:00Z" message: Started container splitter-graph-ebde8 metadata: creationTimestamp: "2026-04-16T18:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:00Z" name: splitter-graph-ebde8-7c98c9c67-6zzq2.18a6eaddab11c663 namespace: kserve-ci-e2e-test resourceVersion: "24929" uid: 89d56f33-3f49-4b5f-ae28-9928730a1181 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ebde8} kind: Pod name: splitter-graph-ebde8-7c98c9c67-6zzq2 namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 48fd4505-0055-468b-911d-1f6feecd7faa kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Stopping container splitter-graph-ebde8 metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:10Z" name: splitter-graph-ebde8-7c98c9c67-6zzq2.18a6eadfe56df525 namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: b5004bbf-4550-45b6-b08c-74d2933d460e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ebde8} kind: Pod name: splitter-graph-ebde8-7c98c9c67-6zzq2 namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 48fd4505-0055-468b-911d-1f6feecd7faa kind: Event lastTimestamp: "2026-04-16T18:49:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:37Z" name: splitter-graph-ebde8-7c98c9c67-6zzq2.18a6eae04cc5bd59 namespace: kserve-ci-e2e-test resourceVersion: "25381" uid: b6657ead-2e7f-4d6c-a740-3bb625deea5a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-ebde8-7c98c9c67 namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: 6d262805-2a55-4257-9ef6-6986e1ee8b61 kind: Event lastTimestamp: "2026-04-16T18:49:00Z" message: 'Created pod: splitter-graph-ebde8-7c98c9c67-6zzq2' metadata: creationTimestamp: "2026-04-16T18:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:00Z" name: splitter-graph-ebde8-7c98c9c67.18a6eadd8dad5c55 namespace: kserve-ci-e2e-test resourceVersion: "24907" uid: 00d6056e-4d9b-40c1-a453-393bb048547c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24900" uid: 616b137d-1045-4003-b862-0eeced4a3b72 kind: Event lastTimestamp: "2026-04-16T18:49:00Z" message: Scaled up replica set splitter-graph-ebde8-7c98c9c67 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:00Z" name: splitter-graph-ebde8.18a6eadd8cf2c6a5 namespace: kserve-ci-e2e-test resourceVersion: "24903" uid: 56944daf-b697-499c-8b0e-fde9ad6204dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24916" uid: e4e06105-9b68-4964-949f-3f2b43fe3b82 kind: Event lastTimestamp: "2026-04-16T18:49:00Z" message: InferenceGraph [splitter-graph-ebde8] is Ready metadata: creationTimestamp: "2026-04-16T18:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:00Z" name: splitter-graph-ebde8.18a6eadd90f0702a namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: ce754fc5-1726-4aa7-9c29-15b81becd24e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:37:05.024436Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-091ad-predictor-bd77f565-lmnzk namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: 6052ebbe-2eda-4f08-b6c5-eef27bd21433 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-091ad-predictor-bd77f565-lmnzk to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:37:05Z" name: success-200-isvc-091ad-predictor-bd77f565-lmnzk.18a6ea370226ecfc namespace: kserve-ci-e2e-test resourceVersion: "18822" uid: 2cffa2c3-f7ac-4477-a7df-61f129bdf902 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-091ad-predictor-bd77f565-lmnzk namespace: kserve-ci-e2e-test resourceVersion: "18837" uid: 6052ebbe-2eda-4f08-b6c5-eef27bd21433 kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:37:05Z" name: success-200-isvc-091ad-predictor-bd77f565-lmnzk.18a6ea370a7d50e1 namespace: kserve-ci-e2e-test resourceVersion: "18861" uid: 45926e4e-d003-461d-bd22-871cf1f13b22 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-091ad-predictor-bd77f565-lmnzk namespace: kserve-ci-e2e-test resourceVersion: "18814" uid: 6052ebbe-2eda-4f08-b6c5-eef27bd21433 kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:05Z" name: success-200-isvc-091ad-predictor-bd77f565-lmnzk.18a6ea370bfdaa72 namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 4a9e58e3-8098-45f3-93e6-741bbd71314c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-091ad-predictor-bd77f565-lmnzk namespace: kserve-ci-e2e-test resourceVersion: "18814" uid: 6052ebbe-2eda-4f08-b6c5-eef27bd21433 kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:05Z" name: success-200-isvc-091ad-predictor-bd77f565-lmnzk.18a6ea3712aeed03 namespace: kserve-ci-e2e-test resourceVersion: "18870" uid: df50fe8e-47e2-4d59-94ed-3f835e3781a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-091ad-predictor-bd77f565-lmnzk namespace: kserve-ci-e2e-test resourceVersion: "18814" uid: 6052ebbe-2eda-4f08-b6c5-eef27bd21433 kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:05Z" name: success-200-isvc-091ad-predictor-bd77f565-lmnzk.18a6ea371325c814 namespace: kserve-ci-e2e-test resourceVersion: "18871" uid: 84bb4763-6709-4ed7-9ef7-b877b9f61d08 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-091ad-predictor-bd77f565-lmnzk namespace: kserve-ci-e2e-test resourceVersion: "18814" uid: 6052ebbe-2eda-4f08-b6c5-eef27bd21433 kind: Event lastTimestamp: "2026-04-16T18:38:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:27Z" name: success-200-isvc-091ad-predictor-bd77f565-lmnzk.18a6ea37444ca9fa namespace: kserve-ci-e2e-test resourceVersion: "19838" uid: 1f9cf02c-0476-4e8e-ba57-226e01342060 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-091ad-predictor-bd77f565-lmnzk namespace: kserve-ci-e2e-test resourceVersion: "18814" uid: 6052ebbe-2eda-4f08-b6c5-eef27bd21433 kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-091ad-predictor-bd77f565-lmnzk.18a6ea49b7a539f1 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 5911582d-4c6a-4c15-ab96-f1166d87b4b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-091ad-predictor-bd77f565 namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: 12b827f2-5dfc-47b7-adda-697595ec39e7 kind: Event lastTimestamp: "2026-04-16T18:37:05Z" message: 'Created pod: success-200-isvc-091ad-predictor-bd77f565-lmnzk' metadata: creationTimestamp: "2026-04-16T18:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:05Z" name: success-200-isvc-091ad-predictor-bd77f565.18a6ea370194a228 namespace: kserve-ci-e2e-test resourceVersion: "18813" uid: c4cfc287-7605-451f-a73a-e64c90f459a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "18791" uid: 1a4439a9-5a2f-4201-967e-b7abe94bd189 kind: Event lastTimestamp: "2026-04-16T18:37:04Z" message: Scaled up replica set success-200-isvc-091ad-predictor-bd77f565 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:04Z" name: success-200-isvc-091ad-predictor.18a6ea36fe364355 namespace: kserve-ci-e2e-test resourceVersion: "18801" uid: 0b83a204-e4cb-440c-a002-3c170c8161ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:37:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "18797" uid: 1ff755c4-1097-4305-8501-213881d47ff0 kind: Event lastTimestamp: "2026-04-16T18:37:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:34Z" name: success-200-isvc-091ad-predictor.18a6ea3a7d0f6318 namespace: kserve-ci-e2e-test resourceVersion: "19113" uid: 3c8b8024-8c47-4c3a-bc9b-ecf2f2c197e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:37:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "18797" uid: 1ff755c4-1097-4305-8501-213881d47ff0 kind: Event lastTimestamp: "2026-04-16T18:37:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:35Z" name: success-200-isvc-091ad-predictor.18a6ea3a7d847fa4 namespace: kserve-ci-e2e-test resourceVersion: "19115" uid: f287af9e-c8d1-45b7-90e4-40e18b4159f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:37:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 1ff755c4-1097-4305-8501-213881d47ff0 kind: Event lastTimestamp: "2026-04-16T18:38:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:37:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:20Z" name: success-200-isvc-091ad-predictor.18a6ea417b30acd2 namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: f23b7814-2e65-4d59-8311-33306723cd95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:37:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-091ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 1ff755c4-1097-4305-8501-213881d47ff0 kind: Event lastTimestamp: "2026-04-16T18:38:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:37:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:20Z" name: success-200-isvc-091ad-predictor.18a6ea417ba829cb namespace: kserve-ci-e2e-test resourceVersion: "19642" uid: 97198487-1a1e-4fb5-a523-f965c064f8e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-091ad namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: 41cc780f-9e20-4e82-a02f-df4a8c63bdc4 kind: Event lastTimestamp: "2026-04-16T18:37:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:57Z" name: success-200-isvc-091ad.18a6ea36fe39800a namespace: kserve-ci-e2e-test resourceVersion: "19423" uid: d8cee32e-05f9-4e8d-80b7-084661cfc205 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-091ad namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: 41cc780f-9e20-4e82-a02f-df4a8c63bdc4 kind: Event lastTimestamp: "2026-04-16T18:37:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-091ad": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-091ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:04Z" name: success-200-isvc-091ad.18a6ea36fed19c9e namespace: kserve-ci-e2e-test resourceVersion: "18805" uid: b138afd7-c1c5-4c88-91ba-63a121048432 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-091ad namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: 41cc780f-9e20-4e82-a02f-df4a8c63bdc4 kind: Event lastTimestamp: "2026-04-16T18:37:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-091ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:04Z" name: success-200-isvc-091ad.18a6ea36fed1dfe4 namespace: kserve-ci-e2e-test resourceVersion: "18806" uid: a697bd70-881c-4fd9-bc03-1038332c56b5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-091ad namespace: kserve-ci-e2e-test resourceVersion: "19421" uid: 41cc780f-9e20-4e82-a02f-df4a8c63bdc4 kind: Event lastTimestamp: "2026-04-16T18:37:57Z" message: InferenceService [success-200-isvc-091ad] is Ready metadata: creationTimestamp: "2026-04-16T18:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:57Z" name: success-200-isvc-091ad.18a6ea4328f6d26d namespace: kserve-ci-e2e-test resourceVersion: "19422" uid: 3dcf1852-a3b9-42fc-ad48-5a46c5433a36 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:37:54.694791Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: 24dda508-a2ca-4338-bff6-5989f30bc584 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg.18a6ea4292bc65e3 namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: 76558535-0b6e-49b8-9861-25eb6562bbe3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg namespace: kserve-ci-e2e-test resourceVersion: "19315" uid: 24dda508-a2ca-4338-bff6-5989f30bc584 kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg.18a6ea429a5cc6e6 namespace: kserve-ci-e2e-test resourceVersion: "19356" uid: 0a5c80fb-ddcb-418e-a940-3ea7db6450fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 24dda508-a2ca-4338-bff6-5989f30bc584 kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg.18a6ea429bb7a1ca namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: a5f42821-dc2b-4c81-934e-00016b2dd0fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 24dda508-a2ca-4338-bff6-5989f30bc584 kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg.18a6ea429ee12df9 namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 8dece15b-b025-41c7-b293-95511879300f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 24dda508-a2ca-4338-bff6-5989f30bc584 kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg.18a6ea429f57a2c7 namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: c673bce9-b61a-4cd9-b38d-c4437abd5947 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:37:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 24dda508-a2ca-4338-bff6-5989f30bc584 kind: Event lastTimestamp: "2026-04-16T18:39:16Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:16Z" name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg.18a6ea42b5af9ced namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: 35a44b98-065d-40d0-8e3c-43dc607bbf36 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 24dda508-a2ca-4338-bff6-5989f30bc584 kind: Event lastTimestamp: "2026-04-16T18:39:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:14Z" name: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg.18a6ea5541bdc88d namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 7b3c232c-187b-4dcc-9761-66aeb3fc16f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0f5d9-predictor-5bc668d579 namespace: kserve-ci-e2e-test resourceVersion: "19289" uid: 419a7ef5-ef3e-4a3a-94ed-54a76622f746 kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: 'Created pod: success-200-isvc-0f5d9-predictor-5bc668d579-ggjbg' metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-0f5d9-predictor-5bc668d579.18a6ea42924d0e22 namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 2c91166a-7eb0-4221-8dea-82a1001cc74b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19284" uid: 1a6ca9d3-b669-4e66-91e8-8404d368f669 kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Scaled up replica set success-200-isvc-0f5d9-predictor-5bc668d579 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-0f5d9-predictor.18a6ea429011134e namespace: kserve-ci-e2e-test resourceVersion: "19294" uid: c030fe29-2e60-4dac-ac6d-f453a1a8b447 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19288" uid: 12065cdb-9f60-42b3-a799-550418e88c85 kind: Event lastTimestamp: "2026-04-16T18:38:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:54Z" name: success-200-isvc-0f5d9-predictor.18a6ea460ea99509 namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: b0b662f2-ed9f-465a-b52e-384a3346a543 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19288" uid: 12065cdb-9f60-42b3-a799-550418e88c85 kind: Event lastTimestamp: "2026-04-16T18:38:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:54Z" name: success-200-isvc-0f5d9-predictor.18a6ea460f19a7be namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: 467393c5-5f50-4b49-849e-c245ab081884 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19522" uid: 12065cdb-9f60-42b3-a799-550418e88c85 kind: Event lastTimestamp: "2026-04-16T18:39:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:09Z" name: success-200-isvc-0f5d9-predictor.18a6ea540ba2d8b0 namespace: kserve-ci-e2e-test resourceVersion: "20226" uid: c512927a-71cb-464c-8396-d3a40c053b7c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0f5d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19522" uid: 12065cdb-9f60-42b3-a799-550418e88c85 kind: Event lastTimestamp: "2026-04-16T18:39:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:09Z" name: success-200-isvc-0f5d9-predictor.18a6ea540c199ab1 namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: a3c52f3a-2257-4ec0-aa61-e17c25c53379 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "19277" uid: fc428c70-b52e-4fb1-b759-319d168d350d kind: Event lastTimestamp: "2026-04-16T18:38:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:46Z" name: success-200-isvc-0f5d9.18a6ea428fe7ffab namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: 7aaead51-220d-4b9c-8785-3a68b2c78776 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "19277" uid: fc428c70-b52e-4fb1-b759-319d168d350d kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: 'Failed to update status for InferenceService "success-200-isvc-0f5d9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0f5d9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-0f5d9.18a6ea4290a4f3cd namespace: kserve-ci-e2e-test resourceVersion: "19300" uid: f74b3b31-6eef-4c0c-8e3c-d1c6c3fbf30f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "19277" uid: fc428c70-b52e-4fb1-b759-319d168d350d kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0f5d9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-0f5d9.18a6ea4290a558ad namespace: kserve-ci-e2e-test resourceVersion: "19304" uid: 184f59be-52a3-481f-b83c-98ed93ba742c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0f5d9 namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: fc428c70-b52e-4fb1-b759-319d168d350d kind: Event lastTimestamp: "2026-04-16T18:38:46Z" message: InferenceService [success-200-isvc-0f5d9] is Ready metadata: creationTimestamp: "2026-04-16T18:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:46Z" name: success-200-isvc-0f5d9.18a6ea4e9b0e6fd4 namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: 7dea803a-a657-4a47-b4eb-915d4c1c8e56 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:26:09.812465Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb namespace: kserve-ci-e2e-test resourceVersion: "13333" uid: a4c3cbdb-b4e6-4cdb-8bb6-dbf94be0c910 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3596a-predictor-7fd7565d7-hcqrb to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:26:09Z" name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb.18a6e99e7479ae3f namespace: kserve-ci-e2e-test resourceVersion: "13339" uid: 7e223103-bf65-4f9b-a259-9a95660ff2ad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb namespace: kserve-ci-e2e-test resourceVersion: "13348" uid: a4c3cbdb-b4e6-4cdb-8bb6-dbf94be0c910 kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:26:09Z" name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb.18a6e99e7cf0e8ef namespace: kserve-ci-e2e-test resourceVersion: "13379" uid: ce00fb3e-35a8-4584-afef-38d37a68a77e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: a4c3cbdb-b4e6-4cdb-8bb6-dbf94be0c910 kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:09Z" name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb.18a6e99e7e575486 namespace: kserve-ci-e2e-test resourceVersion: "13386" uid: 317efb83-79e8-4350-9c39-6346fd1e3a64 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: a4c3cbdb-b4e6-4cdb-8bb6-dbf94be0c910 kind: Event lastTimestamp: "2026-04-16T18:26:23Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" in 13.847s (13.847s including waiting). Image size: 1331750510 bytes.' metadata: creationTimestamp: "2026-04-16T18:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:23Z" name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb.18a6e9a1b7b388fa namespace: kserve-ci-e2e-test resourceVersion: "13602" uid: c7fd7143-80d7-4cd2-9729-b6af250975ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: a4c3cbdb-b4e6-4cdb-8bb6-dbf94be0c910 kind: Event lastTimestamp: "2026-04-16T18:26:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:23Z" name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb.18a6e9a1bc3ffd7d namespace: kserve-ci-e2e-test resourceVersion: "13604" uid: ca05b93a-2b49-4e41-9842-3f6958d39ddc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: a4c3cbdb-b4e6-4cdb-8bb6-dbf94be0c910 kind: Event lastTimestamp: "2026-04-16T18:26:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:23Z" name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb.18a6e9a1bcb1dabe namespace: kserve-ci-e2e-test resourceVersion: "13605" uid: 21a41b14-cda4-462a-89de-281f8c6cb748 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:26:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: a4c3cbdb-b4e6-4cdb-8bb6-dbf94be0c910 kind: Event lastTimestamp: "2026-04-16T18:27:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:05Z" name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb.18a6e9a1d06996c9 namespace: kserve-ci-e2e-test resourceVersion: "13974" uid: 28ef17cd-2833-4885-a6fb-ac73100d7025 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: a4c3cbdb-b4e6-4cdb-8bb6-dbf94be0c910 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb.18a6e9b3764e63bc namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 291a9244-2d4d-4f09-a46d-0533bd9edad2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3596a-predictor-7fd7565d7 namespace: kserve-ci-e2e-test resourceVersion: "13316" uid: 565552fd-bd8b-4e7d-a985-ec403c8ef1fd kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: 'Created pod: success-200-isvc-3596a-predictor-7fd7565d7-hcqrb' metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:09Z" name: success-200-isvc-3596a-predictor-7fd7565d7.18a6e99e73e3d7a9 namespace: kserve-ci-e2e-test resourceVersion: "13334" uid: 4027ef11-c60c-4f94-8fa1-ff83059d55ce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13314" uid: ff5a702b-2271-42e2-b750-41f053f8c678 kind: Event lastTimestamp: "2026-04-16T18:26:09Z" message: Scaled up replica set success-200-isvc-3596a-predictor-7fd7565d7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:09Z" name: success-200-isvc-3596a-predictor.18a6e99e6f0ea65e namespace: kserve-ci-e2e-test resourceVersion: "13318" uid: 256ddd9f-0c13-4890-83ae-aa19a864760c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13317" uid: ffb4b878-c57a-4fdf-968a-5374db0f12f7 kind: Event lastTimestamp: "2026-04-16T18:26:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:54Z" name: success-200-isvc-3596a-predictor.18a6e9a1ee0cc943 namespace: kserve-ci-e2e-test resourceVersion: "13889" uid: 76e45a0a-8078-40a6-ae9f-56cdda681abf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13317" uid: ffb4b878-c57a-4fdf-968a-5374db0f12f7 kind: Event lastTimestamp: "2026-04-16T18:26:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:54Z" name: success-200-isvc-3596a-predictor.18a6e9a1ee875bf0 namespace: kserve-ci-e2e-test resourceVersion: "13890" uid: 2372f00e-e22c-4cd6-a2f6-f02e1ed9e9e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13627" uid: ffb4b878-c57a-4fdf-968a-5374db0f12f7 kind: Event lastTimestamp: "2026-04-16T18:27:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:39Z" name: success-200-isvc-3596a-predictor.18a6e9ac6b157a3e namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: d6525b0c-090e-4537-967a-4ebc448c4983 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3596a-predictor namespace: kserve-ci-e2e-test resourceVersion: "13627" uid: ffb4b878-c57a-4fdf-968a-5374db0f12f7 kind: Event lastTimestamp: "2026-04-16T18:27:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:39Z" name: success-200-isvc-3596a-predictor.18a6e9ac6b891013 namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: af830544-5c3a-4149-917b-a281611543bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3596a namespace: kserve-ci-e2e-test resourceVersion: "13308" uid: f6de498d-ea10-4c19-b241-df462a62db1c kind: Event lastTimestamp: "2026-04-16T18:27:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:15Z" name: success-200-isvc-3596a.18a6e99e6f5b6844 namespace: kserve-ci-e2e-test resourceVersion: "14076" uid: 50fa6ff0-a783-422f-a00e-e0570ea3df51 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3596a namespace: kserve-ci-e2e-test resourceVersion: "14074" uid: f6de498d-ea10-4c19-b241-df462a62db1c kind: Event lastTimestamp: "2026-04-16T18:27:15Z" message: InferenceService [success-200-isvc-3596a] is Ready metadata: creationTimestamp: "2026-04-16T18:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:15Z" name: success-200-isvc-3596a.18a6e9adb4a92170 namespace: kserve-ci-e2e-test resourceVersion: "14075" uid: 1a0c1035-9f9e-4261-8570-2d9ab7306a4c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:49:10.532486Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g namespace: kserve-ci-e2e-test resourceVersion: "25065" uid: 78d71340-9fb6-4558-8eb1-3bf5fcadbba7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-56ec1-predictor-785d9fcb47-mml5g to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g.18a6eadfedcd3388 namespace: kserve-ci-e2e-test resourceVersion: "25070" uid: 7e932fef-5e1f-41e6-9f50-c54c771671e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: 78d71340-9fb6-4558-8eb1-3bf5fcadbba7 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g.18a6eadff604117f namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: 38664cf2-25ec-46c8-bb7d-9e0bcd8fe64a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g namespace: kserve-ci-e2e-test resourceVersion: "25067" uid: 78d71340-9fb6-4558-8eb1-3bf5fcadbba7 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g.18a6eadff7315f5b namespace: kserve-ci-e2e-test resourceVersion: "25118" uid: 6b7d5048-b0aa-4b1b-a148-a5e22bc4070c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g namespace: kserve-ci-e2e-test resourceVersion: "25067" uid: 78d71340-9fb6-4558-8eb1-3bf5fcadbba7 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g.18a6eadffdc0dbe4 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 0a336587-0f59-4f10-9d5f-400005efbfa7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g namespace: kserve-ci-e2e-test resourceVersion: "25067" uid: 78d71340-9fb6-4558-8eb1-3bf5fcadbba7 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g.18a6eadffe40b966 namespace: kserve-ci-e2e-test resourceVersion: "25130" uid: d77a08a8-1c5d-41c5-9dd6-92680cd26d97 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g namespace: kserve-ci-e2e-test resourceVersion: "25067" uid: 78d71340-9fb6-4558-8eb1-3bf5fcadbba7 kind: Event lastTimestamp: "2026-04-16T18:49:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:52Z" name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g.18a6eae017ee11fa namespace: kserve-ci-e2e-test resourceVersion: "25532" uid: d9f18ed2-60ef-495e-87d5-2e4b24ab7151 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:58:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g namespace: kserve-ci-e2e-test resourceVersion: "25067" uid: 78d71340-9fb6-4558-8eb1-3bf5fcadbba7 kind: Event lastTimestamp: "2026-04-16T18:58:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:58:35Z" name: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g.18a6eb6373ab75fc namespace: kserve-ci-e2e-test resourceVersion: "29546" uid: 021613be-e489-4005-b6c7-85eae4e97c9b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-56ec1-predictor-785d9fcb47 namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: 69440e33-4900-4339-af5f-eab1f56dc4e8 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: 'Created pod: success-200-isvc-56ec1-predictor-785d9fcb47-mml5g' metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-56ec1-predictor-785d9fcb47.18a6eadfed70849e namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: d29deea0-4283-4915-87c2-723dce453e67 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25045" uid: 552ebaa8-6b15-4b76-9543-c2cd71ef3490 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Scaled up replica set success-200-isvc-56ec1-predictor-785d9fcb47 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-56ec1-predictor.18a6eadfeb8b9150 namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 5a12bd74-2831-4a5a-900a-cb2bed9eb5bc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: 5c2f2b69-05ba-4b3f-a300-275298a5f398 kind: Event lastTimestamp: "2026-04-16T18:49:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:55Z" name: success-200-isvc-56ec1-predictor.18a6eae36a0a0a03 namespace: kserve-ci-e2e-test resourceVersion: "25564" uid: f375f406-c6ab-46d4-b9c0-a4fe8f37da1d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: 5c2f2b69-05ba-4b3f-a300-275298a5f398 kind: Event lastTimestamp: "2026-04-16T18:49:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:55Z" name: success-200-isvc-56ec1-predictor.18a6eae36a791e40 namespace: kserve-ci-e2e-test resourceVersion: "25565" uid: d75cdedb-d6c6-4d7d-98e2-98615b347ce0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:50:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25274" uid: 5c2f2b69-05ba-4b3f-a300-275298a5f398 kind: Event lastTimestamp: "2026-04-16T18:50:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:25Z" name: success-200-isvc-56ec1-predictor.18a6eaede7250c9f namespace: kserve-ci-e2e-test resourceVersion: "26128" uid: 5a915fb7-95ec-4aca-9f80-d4808eb7e07c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:50:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-56ec1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25274" uid: 5c2f2b69-05ba-4b3f-a300-275298a5f398 kind: Event lastTimestamp: "2026-04-16T18:50:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:25Z" name: success-200-isvc-56ec1-predictor.18a6eaede79cdc51 namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: a44f148d-ce2e-4d92-a645-32196c4cbe96 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25038" uid: 3ada47a4-0847-4246-9dad-3e3c91c72d78 kind: Event lastTimestamp: "2026-04-16T18:50:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:50:02Z" name: success-200-isvc-56ec1.18a6eadfeb317e39 namespace: kserve-ci-e2e-test resourceVersion: "25626" uid: 4b752445-761e-47fb-a47f-ed5c07001b7b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25038" uid: 3ada47a4-0847-4246-9dad-3e3c91c72d78 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-56ec1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-56ec1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-56ec1.18a6eadfebe7dea2 namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 26c0bca3-f418-42d6-a551-bf2c70d57882 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25038" uid: 3ada47a4-0847-4246-9dad-3e3c91c72d78 kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-56ec1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-56ec1.18a6eadfebe82807 namespace: kserve-ci-e2e-test resourceVersion: "25059" uid: 74124ff9-d0fe-4112-9353-2d9cb7fe94c4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:50:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-56ec1 namespace: kserve-ci-e2e-test resourceVersion: "25624" uid: 3ada47a4-0847-4246-9dad-3e3c91c72d78 kind: Event lastTimestamp: "2026-04-16T18:50:02Z" message: InferenceService [success-200-isvc-56ec1] is Ready metadata: creationTimestamp: "2026-04-16T18:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:50:02Z" name: success-200-isvc-56ec1.18a6eaebfc69ca39 namespace: kserve-ci-e2e-test resourceVersion: "25625" uid: 4d1c9517-4e99-4a3f-a01b-a678436668d7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:39:14.934685Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66 namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: 0e13a8de-484c-49a5-afbc-663232b15c46 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5aeb1-predictor-6985d9467-7ss66 to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:39:14Z" name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66.18a6ea554167f502 namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: c70238e8-4687-40c1-8513-b5117fa53932 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66 namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: 0e13a8de-484c-49a5-afbc-663232b15c46 kind: Event lastTimestamp: "2026-04-16T18:39:15Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:39:15Z" name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66.18a6ea5548d09318 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 51c630b0-31e5-4f21-b2b9-bc2cc5302026 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 0e13a8de-484c-49a5-afbc-663232b15c46 kind: Event lastTimestamp: "2026-04-16T18:39:15Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:15Z" name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66.18a6ea554a52eac2 namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 0d14ba4d-0aa5-43f0-83f3-1b7d4ff9152c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 0e13a8de-484c-49a5-afbc-663232b15c46 kind: Event lastTimestamp: "2026-04-16T18:39:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:15Z" name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66.18a6ea555121373e namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: ec1ddd41-2e61-4f10-80a3-ed737e11a3c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 0e13a8de-484c-49a5-afbc-663232b15c46 kind: Event lastTimestamp: "2026-04-16T18:39:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:15Z" name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66.18a6ea55519368b4 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 9c8d6c97-abca-439f-93ce-c1d752445535 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 0e13a8de-484c-49a5-afbc-663232b15c46 kind: Event lastTimestamp: "2026-04-16T18:39:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:56Z" name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66.18a6ea55642753b3 namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 7d163930-acbe-447c-91b2-87e1eebbc1c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 0e13a8de-484c-49a5-afbc-663232b15c46 kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:39Z" name: success-200-isvc-5aeb1-predictor-6985d9467-7ss66.18a6ead8c4a64545 namespace: kserve-ci-e2e-test resourceVersion: "24651" uid: 6dadef5e-c203-4433-a8b8-d80fa12dd9f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5aeb1-predictor-6985d9467 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 40e69112-cf5a-4df1-9527-f51237a2d108 kind: Event lastTimestamp: "2026-04-16T18:39:14Z" message: 'Created pod: success-200-isvc-5aeb1-predictor-6985d9467-7ss66' metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:14Z" name: success-200-isvc-5aeb1-predictor-6985d9467.18a6ea5540f91d36 namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 8df30a55-d182-4e2b-ad37-858b7287a423 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 098a3af1-464e-47eb-876b-9bcdac1f4c34 kind: Event lastTimestamp: "2026-04-16T18:39:14Z" message: Scaled up replica set success-200-isvc-5aeb1-predictor-6985d9467 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:14Z" name: success-200-isvc-5aeb1-predictor.18a6ea553f6bf6d8 namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: 8b50c561-66d1-4c32-bebc-ef7a40d0cd84 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 54e0a970-b0fa-4192-9d0f-2767f0f6d04a kind: Event lastTimestamp: "2026-04-16T18:39:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:29Z" name: success-200-isvc-5aeb1-predictor.18a6ea58be07c6d5 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: cf9e04d8-1012-4424-bf48-4e31316d432b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 54e0a970-b0fa-4192-9d0f-2767f0f6d04a kind: Event lastTimestamp: "2026-04-16T18:39:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:29Z" name: success-200-isvc-5aeb1-predictor.18a6ea58be7a1b9a namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 4727b508-e7d1-4baf-899c-312251c05e2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:39:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 54e0a970-b0fa-4192-9d0f-2767f0f6d04a kind: Event lastTimestamp: "2026-04-16T18:40:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:44Z" name: success-200-isvc-5aeb1-predictor.18a6ea5c3d3d52e3 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 3230d5a8-9aa0-443b-88be-a2c9404ff51f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:39:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5aeb1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 54e0a970-b0fa-4192-9d0f-2767f0f6d04a kind: Event lastTimestamp: "2026-04-16T18:40:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:45Z" name: success-200-isvc-5aeb1-predictor.18a6ea5c3daa69b3 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 32365b7a-916b-4f4e-b7a0-8748465a0a08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: e1a3ad53-d6c6-4e29-b4d9-71c8cc1c8eb4 kind: Event lastTimestamp: "2026-04-16T18:40:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:06Z" name: success-200-isvc-5aeb1.18a6ea553f38babd namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: e8eb335c-aa28-4cc1-b067-ebf784908b33 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5aeb1 namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: e1a3ad53-d6c6-4e29-b4d9-71c8cc1c8eb4 kind: Event lastTimestamp: "2026-04-16T18:40:06Z" message: InferenceService [success-200-isvc-5aeb1] is Ready metadata: creationTimestamp: "2026-04-16T18:40:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:40:06Z" name: success-200-isvc-5aeb1.18a6ea614966b547 namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: db08198b-8fd5-474b-be97-ff9db6705724 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:27:40.068111Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl namespace: kserve-ci-e2e-test resourceVersion: "14369" uid: 64ee8359-c36f-4aea-af5f-1aa8e35a8a5f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6138a-predictor-57b886cd85-wlhjl to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl.18a6e9b378218a3c namespace: kserve-ci-e2e-test resourceVersion: "14375" uid: 692c16dd-7bcd-4732-8813-e3ce738f9137 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: 64ee8359-c36f-4aea-af5f-1aa8e35a8a5f kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl.18a6e9b3806db929 namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: b342e1a4-fc06-4101-adda-b638b8cba42d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 64ee8359-c36f-4aea-af5f-1aa8e35a8a5f kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl.18a6e9b381dfde95 namespace: kserve-ci-e2e-test resourceVersion: "14423" uid: b219c5f2-0647-476f-932a-75469ff75652 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 64ee8359-c36f-4aea-af5f-1aa8e35a8a5f kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl.18a6e9b385f2917f namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: bd5ed765-7cef-47f9-9a4c-10c6da7e91b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 64ee8359-c36f-4aea-af5f-1aa8e35a8a5f kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl.18a6e9b3866469cd namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 8a787805-fbd7-4d1a-9ceb-46b30a11d665 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 64ee8359-c36f-4aea-af5f-1aa8e35a8a5f kind: Event lastTimestamp: "2026-04-16T18:28:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:21Z" name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl.18a6e9b390696917 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 827d08fc-c549-4dc6-a6dc-200831602b0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 64ee8359-c36f-4aea-af5f-1aa8e35a8a5f kind: Event lastTimestamp: "2026-04-16T18:37:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:04Z" name: success-200-isvc-6138a-predictor-57b886cd85-wlhjl.18a6ea36ff4d6710 namespace: kserve-ci-e2e-test resourceVersion: "18808" uid: d9de88b6-ac76-4c9e-b381-1da186380f7f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6138a-predictor-57b886cd85 namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: 86203a24-5c97-4ae9-adb8-f374c13c7f22 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'Created pod: success-200-isvc-6138a-predictor-57b886cd85-wlhjl' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-6138a-predictor-57b886cd85.18a6e9b377a35746 namespace: kserve-ci-e2e-test resourceVersion: "14370" uid: 2338c1a8-f675-4c25-8e12-13c1c1eb83bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: f7d2f2e4-baa8-4cf9-8582-733441433e51 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: Scaled up replica set success-200-isvc-6138a-predictor-57b886cd85 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-6138a-predictor.18a6e9b376270208 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: c4c0b77a-378d-45fe-9205-5f4e99526966 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: 1126ec2c-5a8f-4fff-83a4-bf3061366b5e kind: Event lastTimestamp: "2026-04-16T18:28:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:10Z" name: success-200-isvc-6138a-predictor.18a6e9b6f4c3886f namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 73f9a07a-a083-496f-b923-78e329dd9b2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: 1126ec2c-5a8f-4fff-83a4-bf3061366b5e kind: Event lastTimestamp: "2026-04-16T18:28:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:10Z" name: success-200-isvc-6138a-predictor.18a6e9b6f53811b4 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: 01959e33-d048-4d1b-b059-3609c43b822a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: 1126ec2c-5a8f-4fff-83a4-bf3061366b5e kind: Event lastTimestamp: "2026-04-16T18:28:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:55Z" name: success-200-isvc-6138a-predictor.18a6e9bdf30d05ef namespace: kserve-ci-e2e-test resourceVersion: "15315" uid: 2514782e-5d4e-43cd-89a9-96552159f0a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6138a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: 1126ec2c-5a8f-4fff-83a4-bf3061366b5e kind: Event lastTimestamp: "2026-04-16T18:28:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:55Z" name: success-200-isvc-6138a-predictor.18a6e9bdf3847133 namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: c084e8ea-4a44-4269-9fd4-fb16417f8a5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6138a namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: bfacb11f-0009-42a5-9e8c-5fd031878bf9 kind: Event lastTimestamp: "2026-04-16T18:28:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:31Z" name: success-200-isvc-6138a.18a6e9b375fda90f namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: 8c0a384b-2dc4-42fa-91d3-71886c7efd1b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6138a namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: bfacb11f-0009-42a5-9e8c-5fd031878bf9 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'Failed to update status for InferenceService "success-200-isvc-6138a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6138a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-6138a.18a6e9b3766d587f namespace: kserve-ci-e2e-test resourceVersion: "14366" uid: 75658040-96b1-4f90-a10c-714b487660da reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6138a namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: bfacb11f-0009-42a5-9e8c-5fd031878bf9 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6138a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-6138a.18a6e9b3766d8a35 namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: 0fcc2018-0bd4-4120-a788-a7c4fe6b9bde reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6138a namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: bfacb11f-0009-42a5-9e8c-5fd031878bf9 kind: Event lastTimestamp: "2026-04-16T18:28:31Z" message: InferenceService [success-200-isvc-6138a] is Ready metadata: creationTimestamp: "2026-04-16T18:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:31Z" name: success-200-isvc-6138a.18a6e9bf75f7c911 namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: f7a29ee7-bc2d-4faf-9d4f-c4ba1c2a6bc5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:38:25.359490Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: d97424df-8707-43bc-8e27-47c356e78bc4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-976d9-predictor-85bdbdc997-7jqm9 to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9.18a6ea49b67e8788 namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: f55186ec-d365-43db-816c-ee2cbcca1640 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9 namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: d97424df-8707-43bc-8e27-47c356e78bc4 kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9.18a6ea49be16e762 namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: bbcfe561-38b5-4871-b1a8-19f143e9c242 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9 namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: d97424df-8707-43bc-8e27-47c356e78bc4 kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9.18a6ea49bf6ab481 namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: 389b9713-a707-4fcf-9dcb-2cc670b2eb6a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9 namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: d97424df-8707-43bc-8e27-47c356e78bc4 kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9.18a6ea49c4d60e70 namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: f09f9381-688e-4f30-ac0b-a7a0248930c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9 namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: d97424df-8707-43bc-8e27-47c356e78bc4 kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9.18a6ea49c55ca27d namespace: kserve-ci-e2e-test resourceVersion: "19808" uid: 1767b332-3baa-462a-b41d-ffff0d5c603b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9 namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: d97424df-8707-43bc-8e27-47c356e78bc4 kind: Event lastTimestamp: "2026-04-16T18:39:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:07Z" name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9.18a6ea49f2f725dd namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: d4fced7b-d8f4-4299-9fec-adec551e8678 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9 namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: d97424df-8707-43bc-8e27-47c356e78bc4 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9.18a6eacd3a3faaf0 namespace: kserve-ci-e2e-test resourceVersion: "24133" uid: 2e152e84-526c-4f7f-9ca0-3cad91d6b661 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-976d9-predictor-85bdbdc997 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: 2466d6c2-f515-4742-9a0b-3d9fb20f336b kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: 'Created pod: success-200-isvc-976d9-predictor-85bdbdc997-7jqm9' metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-976d9-predictor-85bdbdc997.18a6ea49b5d7a138 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: ec8daef3-77df-4fe2-b658-22b8105cb2de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 6d01c280-2e44-4c32-802f-f86b3f588087 kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: Scaled up replica set success-200-isvc-976d9-predictor-85bdbdc997 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-976d9-predictor.18a6ea49b3466514 namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: 4fcc48fd-40d2-4efc-be3f-a9ec52b27e77 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 7934b36e-24a6-4853-bbaa-e7658c49e7bb kind: Event lastTimestamp: "2026-04-16T18:38:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:40Z" name: success-200-isvc-976d9-predictor.18a6ea4d3202c94f namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: fb19a38f-9fc3-4520-8724-da335b468f85 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 7934b36e-24a6-4853-bbaa-e7658c49e7bb kind: Event lastTimestamp: "2026-04-16T18:38:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:40Z" name: success-200-isvc-976d9-predictor.18a6ea4d3270797d namespace: kserve-ci-e2e-test resourceVersion: "19942" uid: 119b8501-a296-4f20-ae27-2ece376f7806 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 7934b36e-24a6-4853-bbaa-e7658c49e7bb kind: Event lastTimestamp: "2026-04-16T18:39:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:40Z" name: success-200-isvc-976d9-predictor.18a6ea50b143188e namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: ba790b82-c453-40b8-84f0-82d89737a278 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-976d9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 7934b36e-24a6-4853-bbaa-e7658c49e7bb kind: Event lastTimestamp: "2026-04-16T18:39:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:40Z" name: success-200-isvc-976d9-predictor.18a6ea50b1b8ee4e namespace: kserve-ci-e2e-test resourceVersion: "20661" uid: 1c6235a9-0fab-453e-a357-7d6678f35608 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-976d9 namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: 3a1f62b3-6a76-44bf-b8b2-74f1a9820560 kind: Event lastTimestamp: "2026-04-16T18:39:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:17Z" name: success-200-isvc-976d9.18a6ea49b325a7b3 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 793dc27f-a1e6-4f63-b77d-1bb78b4315d0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-976d9 namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: 3a1f62b3-6a76-44bf-b8b2-74f1a9820560 kind: Event lastTimestamp: "2026-04-16T18:39:17Z" message: InferenceService [success-200-isvc-976d9] is Ready metadata: creationTimestamp: "2026-04-16T18:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:17Z" name: success-200-isvc-976d9.18a6ea55d74d24fe namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 05361219-eb84-4b89-8afb-c537d45fdcb8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:48:39.771217Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc namespace: kserve-ci-e2e-test resourceVersion: "24641" uid: 5db8cb60-43e1-4046-b634-86ce0f17c5d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:48:39Z" name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc.18a6ead8c4498c83 namespace: kserve-ci-e2e-test resourceVersion: "24647" uid: 5f4337c5-19be-4a78-a0bf-397dfd11c5b3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc namespace: kserve-ci-e2e-test resourceVersion: "24656" uid: 5db8cb60-43e1-4046-b634-86ce0f17c5d5 kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:48:39Z" name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc.18a6ead8cc254bc1 namespace: kserve-ci-e2e-test resourceVersion: "24681" uid: 6e4f2763-688c-4590-bd23-8a21e16e2dec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc namespace: kserve-ci-e2e-test resourceVersion: "24644" uid: 5db8cb60-43e1-4046-b634-86ce0f17c5d5 kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:39Z" name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc.18a6ead8cd585455 namespace: kserve-ci-e2e-test resourceVersion: "24685" uid: 181f0bfb-9a70-4a4a-b68e-6f2397cc6e59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc namespace: kserve-ci-e2e-test resourceVersion: "24644" uid: 5db8cb60-43e1-4046-b634-86ce0f17c5d5 kind: Event lastTimestamp: "2026-04-16T18:48:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:40Z" name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc.18a6ead8d1ed9167 namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fa7d1fdb-cd14-45cf-8731-300c75361957 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc namespace: kserve-ci-e2e-test resourceVersion: "24644" uid: 5db8cb60-43e1-4046-b634-86ce0f17c5d5 kind: Event lastTimestamp: "2026-04-16T18:48:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:40Z" name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc.18a6ead8d269ba62 namespace: kserve-ci-e2e-test resourceVersion: "24691" uid: 90d05792-a7d1-43ce-a68a-8a905e06b5e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:48:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc namespace: kserve-ci-e2e-test resourceVersion: "24644" uid: 5db8cb60-43e1-4046-b634-86ce0f17c5d5 kind: Event lastTimestamp: "2026-04-16T19:06:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T19:06:11Z" name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc.18a6ead8da7a8383 namespace: kserve-ci-e2e-test resourceVersion: "32740" uid: 9e331494-e7a0-469b-b6b4-7f5fce6fb4c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T19:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc namespace: kserve-ci-e2e-test resourceVersion: "24644" uid: 5db8cb60-43e1-4046-b634-86ce0f17c5d5 kind: Event lastTimestamp: "2026-04-16T19:06:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T19:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T19:06:09Z" name: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc.18a6ebcd2c8633f4 namespace: kserve-ci-e2e-test resourceVersion: "32695" uid: 3adf7db1-0904-4f8d-a0d7-16b2b413906e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-979d7-predictor-7fc6dfb459 namespace: kserve-ci-e2e-test resourceVersion: "24623" uid: 7e1c8829-4523-4bcd-a2f2-9502fc24d38c kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: 'Created pod: success-200-isvc-979d7-predictor-7fc6dfb459-cs6wc' metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:39Z" name: success-200-isvc-979d7-predictor-7fc6dfb459.18a6ead8c3d4b67e namespace: kserve-ci-e2e-test resourceVersion: "24643" uid: c2694ea8-b558-4bc7-b9ca-13f43a663ebc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24620" uid: 21cba706-fb87-487e-a3e3-2b8f1e848848 kind: Event lastTimestamp: "2026-04-16T18:48:39Z" message: Scaled up replica set success-200-isvc-979d7-predictor-7fc6dfb459 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:39Z" name: success-200-isvc-979d7-predictor.18a6ead8c1c790e7 namespace: kserve-ci-e2e-test resourceVersion: "24627" uid: f6a13285-4e65-478c-8c69-c6b52a1e8085 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24622" uid: b3c1cd39-687e-4b6c-9d31-975fa112d825 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:54Z" name: success-200-isvc-979d7-predictor.18a6eadc40c28b9c namespace: kserve-ci-e2e-test resourceVersion: "24850" uid: d9c6908c-f6f0-4978-97d9-c20d313e2fc7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24622" uid: b3c1cd39-687e-4b6c-9d31-975fa112d825 kind: Event lastTimestamp: "2026-04-16T18:48:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:48:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:54Z" name: success-200-isvc-979d7-predictor.18a6eadc413bcbae namespace: kserve-ci-e2e-test resourceVersion: "24852" uid: fd55e4dc-c198-4cfc-93a6-c7dc13feadfd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:49:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24851" uid: b3c1cd39-687e-4b6c-9d31-975fa112d825 kind: Event lastTimestamp: "2026-04-16T18:49:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:54Z" name: success-200-isvc-979d7-predictor.18a6eadfc00ad8ca namespace: kserve-ci-e2e-test resourceVersion: "25553" uid: f2225d0e-964b-4ac4-bda3-645d3a8f49e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:49:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-979d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "24851" uid: b3c1cd39-687e-4b6c-9d31-975fa112d825 kind: Event lastTimestamp: "2026-04-16T18:49:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:54Z" name: success-200-isvc-979d7-predictor.18a6eadfc077105a namespace: kserve-ci-e2e-test resourceVersion: "25554" uid: 4778af7b-eacf-4eaa-bbec-af635ae680df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:48:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-979d7 namespace: kserve-ci-e2e-test resourceVersion: "24604" uid: babc63b1-e77e-458e-8c47-7dac215e1aa9 kind: Event lastTimestamp: "2026-04-16T18:49:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:31Z" name: success-200-isvc-979d7.18a6ead8c19eb469 namespace: kserve-ci-e2e-test resourceVersion: "25324" uid: b2160a7a-e79d-4fcd-ac80-e909245edebd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-979d7 namespace: kserve-ci-e2e-test resourceVersion: "25322" uid: babc63b1-e77e-458e-8c47-7dac215e1aa9 kind: Event lastTimestamp: "2026-04-16T18:49:31Z" message: InferenceService [success-200-isvc-979d7] is Ready metadata: creationTimestamp: "2026-04-16T18:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:31Z" name: success-200-isvc-979d7.18a6eae4bf1f041a namespace: kserve-ci-e2e-test resourceVersion: "25323" uid: 0f910b3a-060f-4426-bb1c-eb19c0f6c682 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:29.852551Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 04c8be21-b8e6-400f-a327-f677c1e7a15e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:29Z" name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls.18a6e9bf0f83cdb5 namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: 745a2838-3dbb-431b-8704-88fae6dee202 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 04c8be21-b8e6-400f-a327-f677c1e7a15e kind: Event lastTimestamp: "2026-04-16T18:28:29Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:29Z" name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls.18a6e9bf176b0a47 namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: bd2e18ed-33a4-4ea5-81ed-2bfe055a493e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 04c8be21-b8e6-400f-a327-f677c1e7a15e kind: Event lastTimestamp: "2026-04-16T18:28:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:30Z" name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls.18a6e9bf18e1ca2b namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 92cbcadc-d37b-4251-91a6-2f3cf8f65e91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 04c8be21-b8e6-400f-a327-f677c1e7a15e kind: Event lastTimestamp: "2026-04-16T18:28:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:30Z" name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls.18a6e9bf1e41ca12 namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 3c1d34f9-c957-486f-a360-7a3aa32c2f0b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 04c8be21-b8e6-400f-a327-f677c1e7a15e kind: Event lastTimestamp: "2026-04-16T18:28:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:30Z" name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls.18a6e9bf1ebdb8ae namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 6bb2c23d-4295-44ac-b907-cef87d074ee4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 04c8be21-b8e6-400f-a327-f677c1e7a15e kind: Event lastTimestamp: "2026-04-16T18:29:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:11Z" name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls.18a6e9bf3dab38c6 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: dbf0020f-f1a9-4ee6-9c91-529e975edd58 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 04c8be21-b8e6-400f-a327-f677c1e7a15e kind: Event lastTimestamp: "2026-04-16T18:37:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:54Z" name: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls.18a6ea4291b42f24 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 21df21af-5a15-4565-adfd-7ed1c3c72563 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b16e7-predictor-57c5fdffb5 namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 6be12e61-b970-40f9-bdaa-b00551892a9e kind: Event lastTimestamp: "2026-04-16T18:28:29Z" message: 'Created pod: success-200-isvc-b16e7-predictor-57c5fdffb5-gqhls' metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:29Z" name: success-200-isvc-b16e7-predictor-57c5fdffb5.18a6e9bf0effdf35 namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: fd47d87e-6b2b-4381-b1ce-b6fe868c825e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: a17447ad-2350-47e7-ba1d-244ad2415807 kind: Event lastTimestamp: "2026-04-16T18:28:29Z" message: Scaled up replica set success-200-isvc-b16e7-predictor-57c5fdffb5 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:29Z" name: success-200-isvc-b16e7-predictor.18a6e9bf0daa8731 namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: a2e5522d-f035-4fb8-833c-8b93d45e8603 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: e15b3f65-9476-4c29-8487-fb8cb994e36c kind: Event lastTimestamp: "2026-04-16T18:28:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:44Z" name: success-200-isvc-b16e7-predictor.18a6e9c28c3f4b1d namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: 64318cfc-c2f2-455c-bd13-eff39078e0a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: e15b3f65-9476-4c29-8487-fb8cb994e36c kind: Event lastTimestamp: "2026-04-16T18:28:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:44Z" name: success-200-isvc-b16e7-predictor.18a6e9c28cb7fc3f namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 412fd39c-3e4f-4b6e-a042-2ed83577d3e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: e15b3f65-9476-4c29-8487-fb8cb994e36c kind: Event lastTimestamp: "2026-04-16T18:30:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:14Z" name: success-200-isvc-b16e7-predictor.18a6e9c60b86128e namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: e0ff38cd-7dce-4224-9c72-112e3363a668 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b16e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: e15b3f65-9476-4c29-8487-fb8cb994e36c kind: Event lastTimestamp: "2026-04-16T18:30:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:30:14Z" name: success-200-isvc-b16e7-predictor.18a6e9c60bf95aec namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: ca8c2aa6-8094-49d6-ab43-1c497a7a59d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b16e7 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: f0a3e2ee-f3b9-46d8-ab72-90e61b671d02 kind: Event lastTimestamp: "2026-04-16T18:29:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:21Z" name: success-200-isvc-b16e7.18a6e9bf0d7158d4 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 453670b5-44d6-40c5-90c6-9bad2f35e490 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b16e7 namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: f0a3e2ee-f3b9-46d8-ab72-90e61b671d02 kind: Event lastTimestamp: "2026-04-16T18:29:21Z" message: InferenceService [success-200-isvc-b16e7] is Ready metadata: creationTimestamp: "2026-04-16T18:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:21Z" name: success-200-isvc-b16e7.18a6e9cb21d27bc5 namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 5dcc0c2b-40ec-44fa-8495-8482a8c0849d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:47:50.270538Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd namespace: kserve-ci-e2e-test resourceVersion: "24146" uid: bab7994f-ff6b-424b-aeed-96a55fecf97a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ebde8-predictor-67b74b47f-mdvsd to ip-10-0-138-88.ec2.internal metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd.18a6eacd3dd11ec7 namespace: kserve-ci-e2e-test resourceVersion: "24159" uid: 7acf9c86-b0df-4ba2-ab63-3e878ad9ed34 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd namespace: kserve-ci-e2e-test resourceVersion: "24173" uid: bab7994f-ff6b-424b-aeed-96a55fecf97a kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd.18a6eacd4583500f namespace: kserve-ci-e2e-test resourceVersion: "24190" uid: 352764c5-5989-48fa-b31c-9f11e2d98280 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: bab7994f-ff6b-424b-aeed-96a55fecf97a kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd.18a6eacd4727d8c2 namespace: kserve-ci-e2e-test resourceVersion: "24194" uid: cfdc2f3f-fbe7-4097-a030-68ac535e814a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: bab7994f-ff6b-424b-aeed-96a55fecf97a kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd.18a6eacd4b93b405 namespace: kserve-ci-e2e-test resourceVersion: "24199" uid: d7792050-1a2c-4027-aa35-ac9e29265d02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: bab7994f-ff6b-424b-aeed-96a55fecf97a kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd.18a6eacd4c085fc7 namespace: kserve-ci-e2e-test resourceVersion: "24200" uid: 581c6867-b993-45e3-9a8a-81b383f4fbb3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: bab7994f-ff6b-424b-aeed-96a55fecf97a kind: Event lastTimestamp: "2026-04-16T18:49:12Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:12Z" name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd.18a6eacd692e7b6b namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 7bfd243d-bc81-45a7-bf46-0ef5380de265 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: bab7994f-ff6b-424b-aeed-96a55fecf97a kind: Event lastTimestamp: "2026-04-16T18:49:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:10Z" name: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd.18a6eadfedc5aaa7 namespace: kserve-ci-e2e-test resourceVersion: "25071" uid: 504dcb1c-0bfd-4d54-923f-c5fb9430415a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-88.ec2.internal source: component: kubelet host: ip-10-0-138-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ebde8-predictor-67b74b47f namespace: kserve-ci-e2e-test resourceVersion: "24127" uid: fd4b380d-5d73-4580-9571-5318c5fc160f kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: 'Created pod: success-200-isvc-ebde8-predictor-67b74b47f-mdvsd' metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-ebde8-predictor-67b74b47f.18a6eacd3d573684 namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: 90689b34-5d81-47d8-bcea-33ccb12234d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24120" uid: 0c0fc4d5-a45a-4729-b673-d90fbe13d1d3 kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: Scaled up replica set success-200-isvc-ebde8-predictor-67b74b47f from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-ebde8-predictor.18a6eacd39c47351 namespace: kserve-ci-e2e-test resourceVersion: "24130" uid: 78f73fc6-000a-4933-aaec-2dd7f7f9084a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:48:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24126" uid: 36544f0f-6561-467b-9261-782babf83cca kind: Event lastTimestamp: "2026-04-16T18:48:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:20Z" name: success-200-isvc-ebde8-predictor.18a6ead0b880ee23 namespace: kserve-ci-e2e-test resourceVersion: "24439" uid: deea92de-4f8d-48ac-83aa-49b75f8b2d93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:48:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24126" uid: 36544f0f-6561-467b-9261-782babf83cca kind: Event lastTimestamp: "2026-04-16T18:48:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:48:20Z" name: success-200-isvc-ebde8-predictor.18a6ead0b8f72136 namespace: kserve-ci-e2e-test resourceVersion: "24440" uid: afa002e0-cd9b-42e3-8cee-14b46a7b432f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24331" uid: 36544f0f-6561-467b-9261-782babf83cca kind: Event lastTimestamp: "2026-04-16T18:49:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:05Z" name: success-200-isvc-ebde8-predictor.18a6ead7b68e2ca2 namespace: kserve-ci-e2e-test resourceVersion: "24964" uid: 619b2a46-f304-4f22-8b34-1b03a462d407 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ebde8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24331" uid: 36544f0f-6561-467b-9261-782babf83cca kind: Event lastTimestamp: "2026-04-16T18:49:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:05Z" name: success-200-isvc-ebde8-predictor.18a6ead7b6ff2a3b namespace: kserve-ci-e2e-test resourceVersion: "24965" uid: 1cd7ee94-e1aa-4039-b03c-51f55c44dfba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24107" uid: e3e285ee-f90d-4b32-8401-9b9eaeecab5a kind: Event lastTimestamp: "2026-04-16T18:48:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:48:42Z" name: success-200-isvc-ebde8.18a6eacd399e9d14 namespace: kserve-ci-e2e-test resourceVersion: "24741" uid: e9e8d3ab-bae1-4c1d-aa7f-d88b089fbab4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24107" uid: e3e285ee-f90d-4b32-8401-9b9eaeecab5a kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-ebde8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ebde8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-ebde8.18a6eacd3a1d3c3c namespace: kserve-ci-e2e-test resourceVersion: "24132" uid: 950765e8-7b79-4989-a88d-c9a524586bf9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24107" uid: e3e285ee-f90d-4b32-8401-9b9eaeecab5a kind: Event lastTimestamp: "2026-04-16T18:47:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ebde8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:47:50Z" name: success-200-isvc-ebde8.18a6eacd3a1d80f8 namespace: kserve-ci-e2e-test resourceVersion: "24134" uid: 08c5dcec-202c-46ce-911e-f0e127ebb92a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:48:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ebde8 namespace: kserve-ci-e2e-test resourceVersion: "24734" uid: e3e285ee-f90d-4b32-8401-9b9eaeecab5a kind: Event lastTimestamp: "2026-04-16T18:48:42Z" message: InferenceService [success-200-isvc-ebde8] is Ready metadata: creationTimestamp: "2026-04-16T18:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:48:42Z" name: success-200-isvc-ebde8.18a6ead95046d947 namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 12e20d34-1367-4b4b-b128-51076d20ac4a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:27:29.617783Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3596a-5c45cb7657-nxb8c namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: 441d2dc0-7718-4f9d-ace6-9b35dae5182a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3596a-5c45cb7657-nxb8c to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:27:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:27:29Z" name: switch-graph-3596a-5c45cb7657-nxb8c.18a6e9b1093e1fff namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: d728f02d-5b6d-4859-9201-5cc5b7f1fee5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:29Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3596a-5c45cb7657-nxb8c namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 441d2dc0-7718-4f9d-ace6-9b35dae5182a kind: Event lastTimestamp: "2026-04-16T18:27:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-3596a-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:29Z" name: switch-graph-3596a-5c45cb7657-nxb8c.18a6e9b115d716de namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: d77e3eb5-b1ec-40e7-9558-8e47145a58ac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:30Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3596a-5c45cb7657-nxb8c namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: 441d2dc0-7718-4f9d-ace6-9b35dae5182a kind: Event lastTimestamp: "2026-04-16T18:27:30Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:27:30Z" name: switch-graph-3596a-5c45cb7657-nxb8c.18a6e9b145e0f67a namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 987bd900-ec92-4d08-af40-b2de32cdb74b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3596a} kind: Pod name: switch-graph-3596a-5c45cb7657-nxb8c namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 441d2dc0-7718-4f9d-ace6-9b35dae5182a kind: Event lastTimestamp: "2026-04-16T18:27:30Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" metadata: creationTimestamp: "2026-04-16T18:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:30Z" name: switch-graph-3596a-5c45cb7657-nxb8c.18a6e9b147285c3c namespace: kserve-ci-e2e-test resourceVersion: "14228" uid: a72c28b0-3dcc-4144-af08-0cedd5a8b351 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3596a} kind: Pod name: switch-graph-3596a-5c45cb7657-nxb8c namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 441d2dc0-7718-4f9d-ace6-9b35dae5182a kind: Event lastTimestamp: "2026-04-16T18:27:33Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" in 2.464s (2.464s including waiting). Image size: 216207958 bytes.' metadata: creationTimestamp: "2026-04-16T18:27:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:33Z" name: switch-graph-3596a-5c45cb7657-nxb8c.18a6e9b1da0e5e4a namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: fdf377a0-aebd-4f69-9dc2-1031b32000e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3596a} kind: Pod name: switch-graph-3596a-5c45cb7657-nxb8c namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 441d2dc0-7718-4f9d-ace6-9b35dae5182a kind: Event lastTimestamp: "2026-04-16T18:27:33Z" message: 'Created container: switch-graph-3596a' metadata: creationTimestamp: "2026-04-16T18:27:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:33Z" name: switch-graph-3596a-5c45cb7657-nxb8c.18a6e9b1dd0871de namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: e9e4a48c-6dff-4e8d-8da3-0c81f7e650f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3596a} kind: Pod name: switch-graph-3596a-5c45cb7657-nxb8c namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 441d2dc0-7718-4f9d-ace6-9b35dae5182a kind: Event lastTimestamp: "2026-04-16T18:27:33Z" message: Started container switch-graph-3596a metadata: creationTimestamp: "2026-04-16T18:27:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:33Z" name: switch-graph-3596a-5c45cb7657-nxb8c.18a6e9b1dd775efa namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 2f10633d-cc3d-4c67-a9ce-5124e4815c16 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3596a} kind: Pod name: switch-graph-3596a-5c45cb7657-nxb8c namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 441d2dc0-7718-4f9d-ace6-9b35dae5182a kind: Event lastTimestamp: "2026-04-16T18:27:39Z" message: Stopping container switch-graph-3596a metadata: creationTimestamp: "2026-04-16T18:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:39Z" name: switch-graph-3596a-5c45cb7657-nxb8c.18a6e9b36aeaa52b namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: 7a7f2439-2e10-4ccf-823c-0082723add10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:27:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3596a} kind: Pod name: switch-graph-3596a-5c45cb7657-nxb8c namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 441d2dc0-7718-4f9d-ace6-9b35dae5182a kind: Event lastTimestamp: "2026-04-16T18:28:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:09Z" name: switch-graph-3596a-5c45cb7657-nxb8c.18a6e9b4814d661b namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 39b7a110-7dd6-49be-99d5-e1352f5d1635 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3596a-5c45cb7657 namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: c5c05e76-ebfe-42ef-8c2c-641fb722b3ad kind: Event lastTimestamp: "2026-04-16T18:27:29Z" message: 'Created pod: switch-graph-3596a-5c45cb7657-nxb8c' metadata: creationTimestamp: "2026-04-16T18:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:29Z" name: switch-graph-3596a-5c45cb7657.18a6e9b108cd1d25 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: 3cc9af9f-8232-4e5b-8367-d8750b8ac241 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3596a namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: e1e28711-21fa-4558-88e6-d1c25c4c55ae kind: Event lastTimestamp: "2026-04-16T18:27:29Z" message: Scaled up replica set switch-graph-3596a-5c45cb7657 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:29Z" name: switch-graph-3596a.18a6e9b10805e45e namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: d70397c0-0596-466d-a403-24c6988d3245 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3596a namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: 4d6a837a-11bd-4f02-a8b7-d428b666d89e kind: Event lastTimestamp: "2026-04-16T18:27:29Z" message: InferenceGraph [switch-graph-3596a] is Ready metadata: creationTimestamp: "2026-04-16T18:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:29Z" name: switch-graph-3596a.18a6e9b10d166491 namespace: kserve-ci-e2e-test resourceVersion: "14214" uid: e0b61ef4-7f67-4e46-9c58-4842d6b0f4ac reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:50.123514Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6138a-58844c49d4-gm2wj namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: 429e922a-5118-4d2a-8a11-1db6a2def094 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-6138a-58844c49d4-gm2wj to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:50Z" name: switch-graph-6138a-58844c49d4-gm2wj.18a6e9c3c7c21494 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: d18ab555-54c8-4dc7-91d9-ca4c1d00c1f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6138a-58844c49d4-gm2wj namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 429e922a-5118-4d2a-8a11-1db6a2def094 kind: Event lastTimestamp: "2026-04-16T18:28:50Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:50Z" name: switch-graph-6138a-58844c49d4-gm2wj.18a6e9c3e15667ea namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 55c6e1ca-fb7f-4c47-888b-1d7b67490bb2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6138a} kind: Pod name: switch-graph-6138a-58844c49d4-gm2wj namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 429e922a-5118-4d2a-8a11-1db6a2def094 kind: Event lastTimestamp: "2026-04-16T18:28:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:50Z" name: switch-graph-6138a-58844c49d4-gm2wj.18a6e9c3e2c19f87 namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: cc41c4d9-315e-4d86-9caf-987d740e4e6b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6138a} kind: Pod name: switch-graph-6138a-58844c49d4-gm2wj namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 429e922a-5118-4d2a-8a11-1db6a2def094 kind: Event lastTimestamp: "2026-04-16T18:28:50Z" message: 'Created container: switch-graph-6138a' metadata: creationTimestamp: "2026-04-16T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:50Z" name: switch-graph-6138a-58844c49d4-gm2wj.18a6e9c3e5505d24 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: ff924e7a-dcea-4f81-9fe1-8254a8eb08ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6138a} kind: Pod name: switch-graph-6138a-58844c49d4-gm2wj namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 429e922a-5118-4d2a-8a11-1db6a2def094 kind: Event lastTimestamp: "2026-04-16T18:28:50Z" message: Started container switch-graph-6138a metadata: creationTimestamp: "2026-04-16T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:50Z" name: switch-graph-6138a-58844c49d4-gm2wj.18a6e9c3e5bb8c0a namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: ed5d94de-2458-4833-9729-a39474931487 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6138a} kind: Pod name: switch-graph-6138a-58844c49d4-gm2wj namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 429e922a-5118-4d2a-8a11-1db6a2def094 kind: Event lastTimestamp: "2026-04-16T18:37:04Z" message: Stopping container switch-graph-6138a metadata: creationTimestamp: "2026-04-16T18:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:04Z" name: switch-graph-6138a-58844c49d4-gm2wj.18a6ea36f764b250 namespace: kserve-ci-e2e-test resourceVersion: "18777" uid: a9fcc9c6-2f5f-4396-a3a8-36f2bc667a46 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6138a} kind: Pod name: switch-graph-6138a-58844c49d4-gm2wj namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 429e922a-5118-4d2a-8a11-1db6a2def094 kind: Event lastTimestamp: "2026-04-16T18:37:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:31Z" name: switch-graph-6138a-58844c49d4-gm2wj.18a6ea3768aca590 namespace: kserve-ci-e2e-test resourceVersion: "19088" uid: 5c0266b4-1967-4ad5-aa3a-eac3018ed7ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-6138a-58844c49d4 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: a70d38d6-167a-4817-b94d-92ceaa44e331 kind: Event lastTimestamp: "2026-04-16T18:28:50Z" message: 'Created pod: switch-graph-6138a-58844c49d4-gm2wj' metadata: creationTimestamp: "2026-04-16T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:50Z" name: switch-graph-6138a-58844c49d4.18a6e9c3c728ec2c namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 3f49b071-4cd6-49d2-b2d2-d6c1aa908704 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-6138a namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 4a38d5b2-b945-4c9e-89d9-edc690d55017 kind: Event lastTimestamp: "2026-04-16T18:28:50Z" message: Scaled up replica set switch-graph-6138a-58844c49d4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:50Z" name: switch-graph-6138a.18a6e9c3c6298065 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: ac8d8597-b7a7-4378-b47f-2ff9a8449056 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-6138a namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 3fe6049f-5263-4ebc-9fdb-d05a590b7b3e kind: Event lastTimestamp: "2026-04-16T18:28:50Z" message: InferenceGraph [switch-graph-6138a] is Ready metadata: creationTimestamp: "2026-04-16T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:50Z" name: switch-graph-6138a.18a6e9c3cc4cb88b namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: f211b77d-129a-4c2b-ae99-0941c3b39a0d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6138a namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: e89439fc-ef82-432a-87fc-5abdb69cd612 kind: Event lastTimestamp: "2026-04-16T18:29:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:20Z" name: switch-graph-6138a.18a6e9c744ead142 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: f12f046e-1353-4877-a49e-efaa9e4e887e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6138a namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: e89439fc-ef82-432a-87fc-5abdb69cd612 kind: Event lastTimestamp: "2026-04-16T18:29:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:20Z" name: switch-graph-6138a.18a6e9c7456204aa namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: 253bb685-e25b-468c-8ffd-8d2e0dd33316 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:49:49.845417Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-979d7-565d6b4454-s4t4v namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 15bf30bd-7b7e-4f3e-b09b-59fc968da5d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-979d7-565d6b4454-s4t4v to ip-10-0-134-167.ec2.internal metadata: creationTimestamp: "2026-04-16T18:49:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:49:49Z" name: switch-graph-979d7-565d6b4454-s4t4v.18a6eae91508ee0f namespace: kserve-ci-e2e-test resourceVersion: "25498" uid: 80df99a0-d2b8-4e48-8715-32eda80f2815 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d7c555cfd-xvckj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-979d7-565d6b4454-s4t4v namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 15bf30bd-7b7e-4f3e-b09b-59fc968da5d7 kind: Event lastTimestamp: "2026-04-16T18:49:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-979d7-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:50Z" name: switch-graph-979d7-565d6b4454-s4t4v.18a6eae91fcafc95 namespace: kserve-ci-e2e-test resourceVersion: "25509" uid: a1082c3a-7d24-408a-a915-d4c696abaeda reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-979d7-565d6b4454-s4t4v namespace: kserve-ci-e2e-test resourceVersion: "25503" uid: 15bf30bd-7b7e-4f3e-b09b-59fc968da5d7 kind: Event lastTimestamp: "2026-04-16T18:49:50Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:49:50Z" name: switch-graph-979d7-565d6b4454-s4t4v.18a6eae951bbee80 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: f7ffee1e-ebf8-4afd-9418-1f8c45161490 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-979d7} kind: Pod name: switch-graph-979d7-565d6b4454-s4t4v namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 15bf30bd-7b7e-4f3e-b09b-59fc968da5d7 kind: Event lastTimestamp: "2026-04-16T18:49:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3b3b115d46eef70b552cbfe98b8bff65aa58d128941ae255587d75b0e6a21c53" already present on machine metadata: creationTimestamp: "2026-04-16T18:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:50Z" name: switch-graph-979d7-565d6b4454-s4t4v.18a6eae95301d58a namespace: kserve-ci-e2e-test resourceVersion: "25518" uid: 92b3870b-d81b-4e0c-8405-6619def4668f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-979d7} kind: Pod name: switch-graph-979d7-565d6b4454-s4t4v namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 15bf30bd-7b7e-4f3e-b09b-59fc968da5d7 kind: Event lastTimestamp: "2026-04-16T18:49:50Z" message: 'Created container: switch-graph-979d7' metadata: creationTimestamp: "2026-04-16T18:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:50Z" name: switch-graph-979d7-565d6b4454-s4t4v.18a6eae9559dadcf namespace: kserve-ci-e2e-test resourceVersion: "25519" uid: 5bbe8ba0-ffe9-4c82-b1e0-758c42255406 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-979d7} kind: Pod name: switch-graph-979d7-565d6b4454-s4t4v namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 15bf30bd-7b7e-4f3e-b09b-59fc968da5d7 kind: Event lastTimestamp: "2026-04-16T18:49:50Z" message: Started container switch-graph-979d7 metadata: creationTimestamp: "2026-04-16T18:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:49:50Z" name: switch-graph-979d7-565d6b4454-s4t4v.18a6eae95608105d namespace: kserve-ci-e2e-test resourceVersion: "25521" uid: babcefe0-0bd7-4d69-ae3f-7aeb60f40685 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T19:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-979d7} kind: Pod name: switch-graph-979d7-565d6b4454-s4t4v namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 15bf30bd-7b7e-4f3e-b09b-59fc968da5d7 kind: Event lastTimestamp: "2026-04-16T19:06:09Z" message: Stopping container switch-graph-979d7 metadata: creationTimestamp: "2026-04-16T19:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T19:06:09Z" name: switch-graph-979d7-565d6b4454-s4t4v.18a6ebcd2286c9a2 namespace: kserve-ci-e2e-test resourceVersion: "32669" uid: cef9d03b-9d7e-4347-924f-28099e811490 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T19:06:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-979d7} kind: Pod name: switch-graph-979d7-565d6b4454-s4t4v namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 15bf30bd-7b7e-4f3e-b09b-59fc968da5d7 kind: Event lastTimestamp: "2026-04-16T19:06:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T19:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T19:06:32Z" name: switch-graph-979d7-565d6b4454-s4t4v.18a6ebcdd1f7be52 namespace: kserve-ci-e2e-test resourceVersion: "32905" uid: 34e22f6a-93d2-4da6-98bd-339ffc4b75ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-167.ec2.internal source: component: kubelet host: ip-10-0-134-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-979d7-565d6b4454 namespace: kserve-ci-e2e-test resourceVersion: "25490" uid: c5c4f0d5-d2a1-4bff-9622-68519e333b96 kind: Event lastTimestamp: "2026-04-16T18:49:49Z" message: 'Created pod: switch-graph-979d7-565d6b4454-s4t4v' metadata: creationTimestamp: "2026-04-16T18:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:49Z" name: switch-graph-979d7-565d6b4454.18a6eae9147ab496 namespace: kserve-ci-e2e-test resourceVersion: "25494" uid: 944ff829-6949-4c3a-a3b6-5723747e1789 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-979d7 namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: abd7d930-6310-4684-8423-38e7817a65f9 kind: Event lastTimestamp: "2026-04-16T18:49:49Z" message: Scaled up replica set switch-graph-979d7-565d6b4454 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:49:49Z" name: switch-graph-979d7.18a6eae913a33dcb namespace: kserve-ci-e2e-test resourceVersion: "25491" uid: 29eb35c9-140d-4bcf-9498-c6001d1e21b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-979d7 namespace: kserve-ci-e2e-test resourceVersion: "25505" uid: d5f391d5-8b7d-4fef-8e5f-b9612ca32f04 kind: Event lastTimestamp: "2026-04-16T18:49:49Z" message: InferenceGraph [switch-graph-979d7] is Ready metadata: creationTimestamp: "2026-04-16T18:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:49:49Z" name: switch-graph-979d7.18a6eae917cb2bd8 namespace: kserve-ci-e2e-test resourceVersion: "25507" uid: fa7ab5de-a7d4-4bad-ae54-d2a8c615979e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-979d7 namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: c6b3e861-3ec1-47e6-830a-c6bab3d0aaba kind: Event lastTimestamp: "2026-04-16T18:50:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:19Z" name: switch-graph-979d7.18a6eaec924c3d8e namespace: kserve-ci-e2e-test resourceVersion: "25891" uid: 06ed10ba-542a-4524-83d5-cd74e3206aae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-979d7 namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: c6b3e861-3ec1-47e6-830a-c6bab3d0aaba kind: Event lastTimestamp: "2026-04-16T18:50:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:50:19Z" name: switch-graph-979d7.18a6eaec92a2e92c namespace: kserve-ci-e2e-test resourceVersion: "25893" uid: 621aa9cb-aac7-4ecf-8fad-e9b1d7551330 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32911"